chore: release v2.89.0

Revert "fix(dataset_runs): ensure consistent ordering to fetch score data for…"

This reverts commit 2b2022ae85.

.env.dev.example

@@ -44,7 +44,7 @@ S3_FORCE_PATH_STYLE=true
 
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=false
+LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
 LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret

.env.prod.example

@@ -237,7 +237,7 @@ OTEL_SERVICE_NAME="langfuse"
 # CLICKHOUSE_PASSWORD=
 
 # Ingestion
-# LANGFUSE_INGESTION_QUEUE_DELAY_SECONDS=
+# LANGFUSE_INGESTION_QUEUE_DELAY_MS=
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE=
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
 # LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=

.gitignore

@@ -35,9 +35,10 @@ yarn-error.log*
 
 # local env files
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
-.env
-.env.local
-.env*.local
+.env*
+!.env.dev.example
+!.env.local.example
+!.env.prod.example
 
 # vercel
 .vercel

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.87.0",
+  "version": "2.89.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/clickhouse/migrations/0004_drop_observations_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE observations ADD INDEX IF NOT EXISTS idx_project_id project_id TYPE bloom_filter() GRANULARITY 1;

packages/shared/clickhouse/migrations/0004_drop_observations_index.up.sql

@@ -0,0 +1 @@
+ALTER TABLE observations DROP INDEX IF EXISTS idx_project_id;

packages/shared/clickhouse/migrations/0005_add_session_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces DROP INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/migrations/0005_add_session_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_session_id session_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces MATERIALIZE INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/scripts/seed.ts

@@ -5,10 +5,7 @@ import { prepareClickhouse } from "../../scripts/prepareClickhouse";
 
 async function main() {
   try {
-    const projectIds = [
-      "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      "239ad00f-562f-411d-af14-831c75ddd875",
-    ]; // Example project IDs
+    const projectIds = ["7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"]; // Example project IDs
     await prepareClickhouse(projectIds, {
       numberOfDays: 3,
       totalObservations: 1000,

packages/shared/prisma/generated/types.ts

@@ -148,6 +148,7 @@ export type BackgroundMigration = {
     name: string;
     script: string;
     args: unknown;
+    state: Generated<unknown>;
     finished_at: Timestamp | null;
     failed_at: Timestamp | null;
     failed_reason: string | null;

packages/shared/prisma/migrations/20241104111600_background_migrations_add_state_column/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "background_migrations" ADD COLUMN "state" jsonb NOT NULL DEFAULT '{}';

packages/shared/prisma/migrations/20241105110900_add_claude_haiku_35/migration.sql

@@ -0,0 +1,29 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://docs.anthropic.com/en/docs/about-claude/models#model-comparison-table
+  ('cm34aq60d000207ml0j1h31ar', NULL, 'claude-3-5-haiku-20241022', '(?i)^(claude-3-5-haiku-20241022|anthropic\.claude-3-5-haiku-20241022-v1:0|claude-3-5-haiku-V1@20241022)$', NULL, 0.000001, 0.000005, NULL, 'TOKENS', 'claude', NULL),
+  ('cm34aqb9h000307ml6nypd618', NULL, 'claude-3.5-haiku-latest', '(?i)^(claude-3-5-haiku-latest)$', NULL, 0.000001, 0.000005, NULL, 'TOKENS', 'claude', NULL);
+
+INSERT INTO prices (
+  id, 
+  model_id, 
+  usage_type,
+  price
+)
+VALUES
+  ('cm34ax6mc000008jkfqed92mb', 'cm34aq60d000207ml0j1h31ar', 'input', 0.000001),
+  ('cm34axb2o000108jk09wn9b47', 'cm34aqb9h000307ml6nypd618', 'input', 0.000001),
+  ('cm34axeie000208jk8b2ke2t8', 'cm34aq60d000207ml0j1h31ar', 'output', 0.000005),
+  ('cm34axi67000308jk7x1a7qko', 'cm34aqb9h000307ml6nypd618', 'output', 0.000005);

packages/shared/prisma/schema.prisma

@@ -166,6 +166,7 @@ model BackgroundMigration {
   name         String    @unique
   script       String    @map("script")
   args         Json      @map("args")
+  state        Json      @default("{}") @map("state")
   finishedAt   DateTime? @map("finished_at")
   failedAt     DateTime? @map("failed_at")
   failedReason String?   @map("failed_reason")

packages/shared/prisma/seed.ts

@@ -256,7 +256,7 @@ async function main() {
 
     const queueIds = await generateQueuesForProject(
       [project1, project2],
-      configIdsAndNames
+      configIdsAndNames,
     );
 
     const promptIds = await generatePromptsForProject([project1, project2]);
@@ -282,11 +282,11 @@ async function main() {
       project2,
       promptIds,
       queueIds,
-      configIdsAndNames
+      configIdsAndNames,
     );
 
     logger.info(
-      `Seeding ${traces.length} traces, ${observations.length} observations, and ${scores.length} scores`
+      `Seeding ${traces.length} traces, ${observations.length} observations, and ${scores.length} scores`,
     );
 
     await uploadObjects(
@@ -296,7 +296,7 @@ async function main() {
       sessions,
       events,
       comments,
-      queueItems
+      queueItems,
     );
 
     // If openai key is in environment, add it to the projects LLM API keys
@@ -314,7 +314,7 @@ async function main() {
       });
     } else {
       logger.warn(
-        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key."
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key.",
       );
     }
 
@@ -449,7 +449,7 @@ async function main() {
 
         for (const datasetItemId of datasetItemIds) {
           const relevantObservations = observations.filter(
-            (o) => o.projectId === project2.id
+            (o) => o.projectId === project2.id,
           );
           const observation =
             relevantObservations[
@@ -492,7 +492,7 @@ async function uploadObjects(
   sessions: Prisma.TraceSessionCreateManyInput[],
   events: Prisma.ObservationCreateManyInput[],
   comments: Prisma.CommentCreateManyInput[],
-  queueItems: Prisma.AnnotationQueueItemCreateManyInput[]
+  queueItems: Prisma.AnnotationQueueItemCreateManyInput[],
 ) {
   let promises: Prisma.PrismaPromise<unknown>[] = [];
 
@@ -506,14 +506,14 @@ async function uploadObjects(
         },
         create: chunk[0]!,
         update: {},
-      })
+      }),
     );
   });
 
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Sessions ${((i + 1) / promises.length) * 100}% complete`
+        `Seeding of Sessions ${((i + 1) / promises.length) * 100}% complete`,
       );
     await promises[i];
   }
@@ -524,13 +524,13 @@ async function uploadObjects(
     promises.push(
       prisma.trace.createMany({
         data: chunk,
-      })
+      }),
     );
   });
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Traces ${((i + 1) / promises.length) * 100}% complete`
+        `Seeding of Traces ${((i + 1) / promises.length) * 100}% complete`,
       );
     await promises[i];
   }
@@ -540,14 +540,14 @@ async function uploadObjects(
     promises.push(
       prisma.observation.createMany({
         data: chunk,
-      })
+      }),
     );
   });
 
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Observations ${((i + 1) / promises.length) * 100}% complete`
+        `Seeding of Observations ${((i + 1) / promises.length) * 100}% complete`,
       );
     await promises[i];
   }
@@ -557,14 +557,14 @@ async function uploadObjects(
     promises.push(
       prisma.observation.createMany({
         data: chunk,
-      })
+      }),
     );
   });
 
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Events ${((i + 1) / promises.length) * 100}% complete`
+        `Seeding of Events ${((i + 1) / promises.length) * 100}% complete`,
       );
     await promises[i];
   }
@@ -574,13 +574,13 @@ async function uploadObjects(
     promises.push(
       prisma.score.createMany({
         data: chunk,
-      })
+      }),
     );
   });
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Scores ${((i + 1) / promises.length) * 100}% complete`
+        `Seeding of Scores ${((i + 1) / promises.length) * 100}% complete`,
       );
     await promises[i];
   }
@@ -590,13 +590,13 @@ async function uploadObjects(
     promises.push(
       prisma.comment.createMany({
         data: chunk,
-      })
+      }),
     );
   });
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Comments ${((i + 1) / promises.length) * 100}% complete`
+        `Seeding of Comments ${((i + 1) / promises.length) * 100}% complete`,
       );
     await promises[i];
   }
@@ -606,13 +606,13 @@ async function uploadObjects(
     promises.push(
       prisma.annotationQueueItem.createMany({
         data: chunk,
-      })
+      }),
     );
   });
   for (let i = 0; i < promises.length; i++) {
     if (i + 1 >= promises.length || i % Math.ceil(promises.length / 10) === 0)
       logger.info(
-        `Seeding of Annotation Queue Items ${((i + 1) / promises.length) * 100}% complete`
+        `Seeding of Annotation Queue Items ${((i + 1) / promises.length) * 100}% complete`,
       );
     await promises[i];
   }
@@ -634,7 +634,7 @@ function createObjects(
       dataType: ScoreDataType;
       categories: ConfigCategory[] | null;
     }[]
-  >
+  >,
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
@@ -649,7 +649,7 @@ function createObjects(
     // print progress to console with a progress bar that refreshes every 10 iterations
     // random date within last 90 days, with a linear bias towards more recent dates
     const traceTs = new Date(
-      Date.now() - Math.floor(Math.random() ** 1.5 * 90 * 24 * 60 * 60 * 1000)
+      Date.now() - Math.floor(Math.random() ** 1.5 * 90 * 24 * 60 * 60 * 1000),
     );
 
     const envTag = envTags[Math.floor(Math.random() * envTags.length)];
@@ -805,11 +805,11 @@ function createObjects(
     for (let j = 0; j < Math.floor(Math.random() * 10) + 1; j++) {
       // add between 1 and 30 ms to trace timestamp
       const spanTsStart = new Date(
-        traceTs.getTime() + Math.floor(Math.random() * 30)
+        traceTs.getTime() + Math.floor(Math.random() * 30),
       );
       // random duration of upto 5000ms
       const spanTsEnd = new Date(
-        spanTsStart.getTime() + Math.floor(Math.random() * 5000)
+        spanTsStart.getTime() + Math.floor(Math.random() * 5000),
       );
 
       const span = {
@@ -844,22 +844,22 @@ function createObjects(
         const generationTsStart = new Date(
           spanTsStart.getTime() +
             Math.floor(
-              Math.random() * (spanTsEnd.getTime() - spanTsStart.getTime())
-            )
+              Math.random() * (spanTsEnd.getTime() - spanTsStart.getTime()),
+            ),
         );
         const generationTsEnd = new Date(
           generationTsStart.getTime() +
             Math.floor(
               Math.random() *
-                (spanTsEnd.getTime() - generationTsStart.getTime())
-            )
+                (spanTsEnd.getTime() - generationTsStart.getTime()),
+            ),
         );
         // somewhere in the middle
         const generationTsCompletionStart = new Date(
           generationTsStart.getTime() +
             Math.floor(
-              (generationTsEnd.getTime() - generationTsStart.getTime()) / 3
-            )
+              (generationTsEnd.getTime() - generationTsStart.getTime()) / 3,
+            ),
         );
 
         const promptTokens = Math.floor(Math.random() * 1000) + 300;
@@ -880,7 +880,7 @@ function createObjects(
         const promptId =
           promptIds.get(projectId)![
             Math.floor(
-              Math.random() * Math.floor(promptIds.get(projectId)!.length / 2)
+              Math.random() * Math.floor(promptIds.get(projectId)!.length / 2),
             )
           ];
 
@@ -962,8 +962,8 @@ function createObjects(
           const eventTs = new Date(
             spanTsStart.getTime() +
               Math.floor(
-                Math.random() * (spanTsEnd.getTime() - spanTsStart.getTime())
-              )
+                Math.random() * (spanTsEnd.getTime() - spanTsStart.getTime()),
+              ),
           );
 
           events.push({
@@ -985,7 +985,7 @@ function createObjects(
   }
   // find unique sessions by id and projectid
   const uniqueSessions: Prisma.TraceSessionCreateManyInput[] = Array.from(
-    new Set(sessions.map((session) => JSON.stringify(session)))
+    new Set(sessions.map((session) => JSON.stringify(session))),
   ).map((session) => JSON.parse(session) as Prisma.TraceSessionCreateManyInput);
 
   return {
@@ -1007,7 +1007,7 @@ async function generatePromptsForProject(projects: Project[]) {
     projects.map(async (project) => {
       const promptIdsForProject = await generatePrompts(project);
       promptIds.set(project.id, promptIdsForProject);
-    })
+    }),
   );
   return promptIds;
 }
@@ -1193,7 +1193,7 @@ async function generateConfigsForProject(projects: Project[]) {
     projects.map(async (project) => {
       const configNameAndId = await generateConfigs(project);
       projectIdsToConfigs.set(project.id, configNameAndId);
-    })
+    }),
   );
   return projectIdsToConfigs;
 }
@@ -1343,7 +1343,7 @@ async function generateQueuesForProject(
       dataType: ScoreDataType;
       categories: ConfigCategory[] | null;
     }[]
-  >
+  >,
 ) {
   const projectIdsToQueues: Map<string, string[]> = new Map();
 
@@ -1351,10 +1351,10 @@ async function generateQueuesForProject(
     projects.map(async (project) => {
       const queueIds = await generateQueues(
         project,
-        configIdsAndNames.get(project.id) ?? []
+        configIdsAndNames.get(project.id) ?? [],
       );
       projectIdsToQueues.set(project.id, queueIds);
-    })
+    }),
   );
   return projectIdsToQueues;
 }
@@ -1366,7 +1366,7 @@ async function generateQueues(
     id: string;
     dataType: ScoreDataType;
     categories: ConfigCategory[] | null;
-  }[]
+  }[],
 ) {
   const queue = {
     id: `queue-${v4()}`,

packages/shared/scripts/load-seed.ts

@@ -13,7 +13,7 @@ const createRandomProjectId = () => randomUUID().toString();
 
 const prepareProjectsAndApiKeys = async (
   numOfProjects: number,
-  opts: { requiredProjectIds: string[] }
+  opts: { requiredProjectIds: string[] },
 ) => {
   const { requiredProjectIds } = opts;
   const projectsToCreate = numOfProjects - requiredProjectIds.length;
@@ -51,7 +51,7 @@ const prepareProjectsAndApiKeys = async (
     });
     if (!apiKeyExists) {
       const sk = await hashSecretKey(
-        `sk-${Math.random().toString(36).substr(2, 9)}`
+        `sk-${Math.random().toString(36).substr(2, 9)}`,
       );
       await prisma.apiKey.create({
         data: {
@@ -79,29 +79,34 @@ async function main() {
   let numberOfDays = parseInt(process.argv[3], 10);
   let totalObservations = parseInt(process.argv[4], 10);
 
+  logger.info(process.argv);
+  logger.info(
+    `Preparing Clickhouse for ${numOfProjects} projects and ${numberOfDays} days with max Observations ${totalObservations}.`,
+  );
+
   if (isNaN(totalObservations)) {
     logger.warn(
-      "Total observations not provided or invalid. Defaulting to 1000 observations."
+      "Total observations not provided or invalid. Defaulting to 1000 observations.",
     );
     totalObservations = 1000;
   }
 
   if (isNaN(numOfProjects)) {
     logger.warn(
-      "Number of projects not provided or invalid. Defaulting to 10 projects."
+      "Number of projects not provided or invalid. Defaulting to 10 projects.",
     );
     numOfProjects = 10;
   }
 
   if (isNaN(numberOfDays)) {
     logger.warn(
-      "Number of days not provided or invalid. Defaulting to 3 days."
+      "Number of days not provided or invalid. Defaulting to 3 days.",
     );
     numberOfDays = 3;
   }
 
   logger.info(
-    `Preparing Clickhouse for ${numOfProjects} projects and ${numberOfDays} days with max Observations ${totalObservations}.`
+    `Preparing Clickhouse for ${numOfProjects} projects and ${numberOfDays} days with max Observations ${totalObservations}.`,
   );
 
   try {

packages/shared/scripts/prepareClickhouse.ts

@@ -1,3 +1,4 @@
+import { prisma } from "../src/db";
 import { clickhouseClient, logger } from "../src/server";
 
 function randn_bm(min: number, max: number, skew: number) {
@@ -60,7 +61,7 @@ export const prepareClickhouse = async (
     SELECT toString(number) AS id,
       toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS timestamp,
       concat('name_', toString(rand() % 100)) AS name,
-      concat('user_id_', toString(randUniform(0, 100))) AS user_id,
+      concat('user_id_', toInt64(randExponential(1 / 100))) AS user_id,
       map('key', 'value') AS metadata,
       concat('release_', toString(randUniform(0, 100))) AS release,
       concat('version_', toString(randUniform(0, 100))) AS version,
@@ -70,7 +71,7 @@ export const prepareClickhouse = async (
       array('tag1', 'tag2') as tags,
       repeat('input', toInt64(randExponential(1 / 100))) AS input,
       repeat('output', toInt64(randExponential(1 / 100))) AS output,
-      concat('session_', toString(rand() % 100)) AS session_id,
+      if(randUniform(0, 1) < 0.2, NULL, concat('session_', toString(rand() % 1000))) AS session_id,
       timestamp AS created_at,
       timestamp AS updated_at,
       timestamp AS event_ts,
@@ -83,13 +84,13 @@ export const prepareClickhouse = async (
     SELECT toString(number) AS id,
       toString(floor(randUniform(0, ${tracesPerProject}))) AS trace_id,
       '${projectId}' AS project_id,
-      if(rand() < 0.47, 'GENERATION', if(rand() < 0.94, 'SPAN', 'EVENT')) AS type,
+      if(randUniform(0, 1) < 0.47, 'GENERATION', if(randUniform(0, 1) < 0.94, 'SPAN', 'EVENT')) AS type,
       toString(rand()) AS parent_observation_id,
       toDateTime(now() - randUniform(0, ${opts.numberOfDays} * 24 * 60 * 60)) AS start_time,
       addSeconds(start_time, if(rand() < 0.6, floor(randUniform(0, 20)), floor(randUniform(0, 3600)))) AS end_time,
       concat('name', toString(rand() % 100)) AS name,
       map('key', 'value') AS metadata,
-      if(rand() < 0.9, 'DEFAULT', if(rand() < 0.5, 'ERROR', if(rand() < 0.5, 'DEBUG', 'WARNING'))) AS level,
+      if(randUniform(0, 1) < 0.9, 'DEFAULT', if(randUniform(0, 1) < 0.5, 'ERROR', if(randUniform(0, 1) < 0.5, 'DEBUG', 'WARNING'))) AS level,
       'status_message' AS status_message,
       'version' AS version,
       repeat('input', toInt64(randExponential(1 / 100))) AS input,
@@ -102,7 +103,7 @@ export const prepareClickhouse = async (
         when number % 2 = 0 then 'cltr0w45b000008k1407o9qv1'
         else 'clrntkjgy000f08jx79v9g1xj'
       end as internal_model_id,
-      '{"temperature": 0.7, "max_tokens": 15d0}' AS model_parameters,
+      '{"temperature": 0.7, "max_tokens": 150}' AS model_parameters,
       map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS provided_usage_details,
       map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS usage_details,
       map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS provided_cost_details,
@@ -130,7 +131,7 @@ export const prepareClickhouse = async (
         toString(floor(randUniform(0, ${observationsPerProject}))),
         NULL
       ) AS observation_id,
-      concat('name_', toString(rand() % 100)) AS name,
+      concat('name_', toString(rand() % 10)) AS name,
       randUniform(0, 100) as value,
       'API' as source,
       'comment' as comment,
@@ -157,6 +158,34 @@ export const prepareClickhouse = async (
         },
       });
     }
+    // we also need to upsert trace sessions in postgres
+
+    const sessionQuery = `
+      SELECT session_id, project_id
+      FROM traces 
+      WHERE session_id IS NOT NULL;
+    `;
+    const sessionResult = await clickhouseClient.query({
+      query: sessionQuery,
+      format: "JSONEachRow",
+    });
+
+    const sessionData = await sessionResult.json<{
+      session_id: string;
+      project_id: string;
+    }>();
+
+    const idProjectIdCombinations = sessionData.map((session) => ({
+      id: session.session_id,
+      projectId: session.project_id,
+      public: Math.random() < 0.1,
+      bookmarked: Math.random() < 0.1,
+    }));
+
+    await prisma.traceSession.createMany({
+      data: idProjectIdCombinations,
+      skipDuplicates: true,
+    });
   }
 
   const tables = ["traces", "scores", "observations"];

packages/shared/src/db.ts

@@ -76,7 +76,7 @@ export class KyselySingleton {
               createQueryCompiler: () => new PostgresQueryCompiler(),
             },
           }),
-      })
+      }),
     );
 
     return KyselySingleton.instance;
@@ -103,7 +103,7 @@ if (process.env.NODE_ENV === "development") {
             createQueryCompiler: () => new PostgresQueryCompiler(),
           },
         }),
-    })
+    }),
   );
 }
 

packages/shared/src/env.ts

@@ -38,6 +38,16 @@ const EnvSchema = z.object({
   ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_S3_EVENT_UPLOAD_ENABLED: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_PREFIX: z.string().default(""),
+  LANGFUSE_S3_EVENT_UPLOAD_REGION: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
+    .enum(["true", "false"])
+    .default("false"),
 });
 
 export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/features/scores/index.ts

@@ -0,0 +1,3 @@
+export * from "./scoreTypes";
+export * from "./types";
+export * from "./scoreConfigTypes";

packages/shared/src/features/scores/types.ts

@@ -0,0 +1,37 @@
+import { ScoreDataType, ScoreSource } from "@prisma/client";
+
+export type CategoricalAggregate = {
+  type: "CATEGORICAL";
+  values: string[];
+  valueCounts: { value: string; count: number }[];
+  comment?: string | null;
+};
+
+export type NumericAggregate = {
+  type: "NUMERIC";
+  values: number[];
+  average: number;
+  comment?: string | null;
+};
+
+export type ScoreAggregate = Record<
+  string,
+  CategoricalAggregate | NumericAggregate
+>;
+
+export type ScoreSimplified = {
+  name: string;
+  dataType: ScoreDataType;
+  source: ScoreSource;
+  value?: number | null;
+  comment?: string | null;
+  stringValue?: string | null;
+};
+
+export type LastUserScore = ScoreSimplified & {
+  timestamp: string;
+  traceId: string;
+  observationId?: string | null;
+
+  userId: string;
+};

packages/shared/src/index.ts

@@ -28,8 +28,7 @@ export * from "./features/batchExport/types";
 export * from "./features/annotation/types";
 
 // scores
-export * from "./features/scores/scoreConfigTypes";
-export * from "./features/scores/scoreTypes";
+export * from "./features/scores";
 
 // comments
 export * from "./features/comments/types";

packages/shared/src/server/clickhouse/client.ts

@@ -13,3 +13,12 @@ export const clickhouseClient = createClient({
     wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
   },
 });
+
+/**
+ * Accepts a JavaScript date and returns the DateTime in format YYYY-MM-DD HH:MM:SS
+ */
+
+export const convertDateToClickhouseDateTime = (date: Date): string => {
+  // 2024-11-06T20:37:00.000Z -> 2024-11-06 21:37:00
+  return date.toISOString().slice(0, 19).replace("T", " ");
+};

packages/shared/src/server/index.ts

@@ -10,7 +10,7 @@ export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
 export * from "./utils/transforms";
 export * from "./clickhouse/client";
-export * from "./clickhouse/schema-utils";
+export * from "./clickhouse/schemaUtils";
 export * from "./clickhouse/schema";
 export * from "./repositories/definitions";
 export * from "../server/ingestion/types";

packages/shared/src/server/ingestion/legacy/EventProcessor.ts

@@ -367,7 +367,7 @@ export class ObservationProcessor implements EventProcessor {
           text: body.input,
         });
       } else {
-        logger.info(
+        logger.debug(
           `No input provided, trying to calculate for id: ${existingObservation?.id}`,
         );
         const observationInput = await prisma.observation.findFirst({
@@ -393,7 +393,7 @@ export class ObservationProcessor implements EventProcessor {
           text: body.output,
         });
       } else {
-        logger.info(
+        logger.debug(
           `No output provided, trying to calculate for id: ${existingObservation?.id}`,
         );
         const observationOutput = await prisma.observation.findFirst({

packages/shared/src/server/ingestion/legacy/index.ts

@@ -77,7 +77,7 @@ export const handleBatch = async (
       // Decide how to handle the error: rethrow, continue, or push an error object to results
       // For example, push an error object:
       errors.push({
-        error: error,
+        error,
         id: singleEvent.id,
         type: singleEvent.type,
       });
@@ -122,7 +122,7 @@ const handleSingleEvent = async (
     restEvent = rest;
   }
 
-  logger.info(
+  logger.debug(
     `handling single event ${event.id} of type ${event.type}:  ${JSON.stringify({ body: restEvent })}`,
   );
 
@@ -188,7 +188,7 @@ export const addTracesToTraceUpsertQueue = async (
 
   try {
     if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && redis) {
-      logger.info(`Sending ${traceEvents.length} events to worker via Redis`);
+      logger.debug(`Sending ${traceEvents.length} events to worker via Redis`);
 
       const queue = TraceUpsertQueue.getInstance();
       if (!queue) {

packages/shared/src/server/ingestion/types.ts

@@ -3,7 +3,7 @@ import { z } from "zod";
 
 import { NonEmptyString, jsonSchema } from "../../utils/zod";
 import { ModelUsageUnit } from "../../constants";
-import { ObservationLevel } from "@prisma/client";
+import { ObservationLevel, ScoreSource } from "@prisma/client";
 
 export const Usage = z.object({
   input: z.number().int().nullish(),
@@ -163,6 +163,7 @@ const BaseScoreBody = z.object({
   traceId: z.string(),
   observationId: z.string().nullish(),
   comment: z.string().nullish(),
+  source: z.nativeEnum(ScoreSource).default(ScoreSource.API),
 });
 
 /**

packages/shared/src/server/ingestion/validateAndInflateScore.ts

@@ -76,7 +76,7 @@ function inflateScoreBody(
   const { body, projectId, scoreId, config } = params;
 
   const relevantDataType = config?.dataType ?? body.dataType;
-  const scoreProps = { ...body, id: scoreId, projectId, source: "API" };
+  const scoreProps = { source: "API", ...body, id: scoreId, projectId };
 
   if (typeof body.value === "number") {
     if (relevantDataType && relevantDataType === ScoreDataType.BOOLEAN) {

packages/shared/src/server/instrumentation/index.ts

@@ -38,7 +38,7 @@ export async function instrumentAsync<T>(
   return getTracer(ctx.traceScope ?? callback.name).startActiveSpan(
     ctx.name,
     {
-      root: !Boolean(ctx.traceContext) && ctx.rootSpan,
+      root: !ctx.traceContext && ctx.rootSpan,
       kind: ctx.spanKind,
     },
     activeContext,
@@ -72,7 +72,7 @@ export function instrumentSync<T>(
   return getTracer(ctx.traceScope ?? callback.name).startActiveSpan(
     ctx.name,
     {
-      root: !Boolean(ctx.traceContext) && ctx.rootSpan,
+      root: !ctx.traceContext && ctx.rootSpan,
       kind: ctx.spanKind,
     },
     activeContext,

packages/shared/src/server/llm/types.ts

@@ -49,7 +49,7 @@ export const ZodModelConfig = z.object({
   top_p: z.coerce.number().optional(),
 });
 
-// NOTE: Update docs page when changing this!
+// NOTE: Update docs page when changing this! https://langfuse.com/docs/playground#openai-playground--anthropic-playground
 export const openAIModels = [
   "gpt-4o",
   "gpt-4o-2024-08-06",
@@ -76,11 +76,13 @@ export const openAIModels = [
 
 export type OpenAIModel = (typeof openAIModels)[number];
 
-// NOTE: Update docs page when changing this!
+// NOTE: Update docs page when changing this! https://langfuse.com/docs/playground#openai-playground--anthropic-playground
 export const anthropicModels = [
+  "claude-3-5-sonnet-20241022",
   "claude-3-5-sonnet-20240620",
   "claude-3-opus-20240229",
   "claude-3-sonnet-20240229",
+  "claude-3-5-haiku-20241022",
   "claude-3-haiku-20240307",
   "claude-2.1",
   "claude-2.0",

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -14,6 +14,8 @@ function randomCharacters() {
 export interface Filter {
   apply(): ClickhouseFilter;
   clickhouseTable: string;
+  operator: (typeof filterOperators)[keyof typeof filterOperators][number];
+  field: string;
 }
 type ClickhouseFilter = {
   query: string;
@@ -22,9 +24,9 @@ type ClickhouseFilter = {
 
 export class StringFilter implements Filter {
   public clickhouseTable: string;
-  protected field: string;
-  protected value: string;
-  protected operator: (typeof filterOperators)["string"][number];
+  public field: string;
+  public value: string;
+  public operator: (typeof filterOperators)["string"][number];
   protected tablePrefix?: string;
 
   constructor(opts: {
@@ -42,7 +44,7 @@ export class StringFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varName = `stringFilter${this.field}`;
+    const varName = `stringFilter${randomCharacters()}`;
 
     const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
     let query: string;
@@ -75,9 +77,9 @@ export class StringFilter implements Filter {
 
 export class NumberFilter implements Filter {
   public clickhouseTable: string;
-  protected field: string;
-  protected value: number;
-  protected operator: (typeof filterOperators)["number"][number];
+  public field: string;
+  public value: number;
+  public operator: (typeof filterOperators)["number"][number];
   protected tablePrefix?: string;
 
   constructor(opts: {
@@ -106,9 +108,9 @@ export class NumberFilter implements Filter {
 
 export class DateTimeFilter implements Filter {
   public clickhouseTable: string;
-  protected field: string;
-  protected value: Date;
-  protected operator: (typeof filterOperators)["datetime"][number];
+  public field: string;
+  public value: Date;
+  public operator: (typeof filterOperators)["datetime"][number];
   protected tablePrefix?: string;
 
   constructor(opts: {
@@ -137,9 +139,9 @@ export class DateTimeFilter implements Filter {
 
 export class StringOptionsFilter implements Filter {
   public clickhouseTable: string;
-  protected field: string;
-  protected values: string[];
-  protected operator: (typeof filterOperators.stringOptions)[number];
+  public field: string;
+  public values: string[];
+  public operator: (typeof filterOperators.stringOptions)[number];
   protected tablePrefix?: string;
 
   constructor(opts: {
@@ -169,12 +171,72 @@ export class StringOptionsFilter implements Filter {
   }
 }
 
+// stringObject filter is used when we want to filter on a key value pair in a clickhouse map.
+// As we use the MAP form clickhouse, we can only filter efficiently on the first level of a json obj.
+export class StringObjectFilter implements Filter {
+  public clickhouseTable: string;
+  public field: string;
+  public key: string;
+  public value: string;
+  public operator: (typeof filterOperators)["stringObject"][number];
+  protected tablePrefix?: string;
+
+  constructor(opts: {
+    clickhouseTable: string;
+    field: string;
+    operator: (typeof filterOperators)["stringObject"][number];
+    key: string;
+    value: string;
+    tablePrefix?: string;
+  }) {
+    this.clickhouseTable = opts.clickhouseTable;
+    this.field = opts.field;
+    this.value = opts.value;
+    this.operator = opts.operator;
+    this.tablePrefix = opts.tablePrefix;
+    this.key = opts.key;
+  }
+
+  apply(): ClickhouseFilter {
+    const varKeyName = `stringObjectKeyFilter${randomCharacters()}`;
+    const varValueName = `stringObjectValueFilter${randomCharacters()}`;
+    const column = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
+
+    //  const query: `${column}['{varKeyName: String}'] ${this.operator} {${varValueName}: String}`,
+    let query: string;
+    switch (this.operator) {
+      case "=":
+        query = `${column}[{${varKeyName}: String}] = {${varValueName}: String}`;
+        break;
+      case "contains":
+        query = `position(${column}[{${varKeyName}: String}], {${varValueName}: String}) > 0`;
+        break;
+      case "does not contain":
+        query = `position(${column}[{${varKeyName}: String}], {${varValueName}: String}) = 0`;
+        break;
+      case "starts with":
+        query = `startsWith(${column}[{${varKeyName}: String}], {${varValueName}: String})`;
+        break;
+      case "ends with":
+        query = `endsWith(${column}[{${varKeyName}: String}], {${varValueName}: String})`;
+        break;
+      default:
+        throw new Error(`Unsupported operator: ${this.operator}`);
+    }
+
+    return {
+      query,
+      params: { [varKeyName]: this.key, [varValueName]: this.value },
+    };
+  }
+}
+
 // this is used when we want to filter multiple values on a clickhouse column which is also an array
 export class ArrayOptionsFilter implements Filter {
   public clickhouseTable: string;
-  protected field: string;
-  protected values: string[];
-  protected operator: (typeof filterOperators.arrayOptions)[number];
+  public field: string;
+  public values: string[];
+  public operator: (typeof filterOperators.arrayOptions)[number];
   protected tablePrefix?: string;
 
   constructor(opts: {
@@ -219,10 +281,10 @@ export class ArrayOptionsFilter implements Filter {
 
 export class NumberObjectFilter implements Filter {
   public clickhouseTable: string;
-  protected field: string;
-  protected key: string;
-  protected value: number;
-  protected operator: (typeof filterOperators)["numberObject"][number];
+  public field: string;
+  public key: string;
+  public value: number;
+  public operator: (typeof filterOperators)["numberObject"][number];
   protected tablePrefix?: string;
 
   constructor(opts: {
@@ -254,13 +316,15 @@ export class NumberObjectFilter implements Filter {
 
 export class BooleanFilter implements Filter {
   public clickhouseTable: string;
-  protected field: string;
-  protected value: boolean;
+  public field: string;
+  public operator: (typeof filterOperators)["boolean"][number];
+  public value: boolean;
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
+    operator: (typeof filterOperators)["boolean"][number];
     value: boolean;
     tablePrefix?: string;
   }) {
@@ -268,13 +332,14 @@ export class BooleanFilter implements Filter {
     this.field = opts.field;
     this.value = opts.value;
     this.tablePrefix = opts.tablePrefix;
+    this.operator = opts.operator;
   }
 
   apply(): ClickhouseFilter {
     const uid = randomCharacters();
     const varName = `booleanFilter${uid}`;
     return {
-      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} = {${varName}: Boolean}`,
+      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: Boolean}`,
       params: { [varName]: this.value },
     };
   }
@@ -291,6 +356,10 @@ export class FilterList {
     this.filters.push(...filter);
   }
 
+  find(predicate: (filter: Filter) => boolean) {
+    return this.filters.find(predicate);
+  }
+
   public apply(): ClickhouseFilter {
     if (this.filters.length === 0) {
       return {

packages/shared/src/server/queries/clickhouse-sql/factory.ts

@@ -1,7 +1,7 @@
 import z from "zod";
 import { singleFilter } from "../../../interfaces/filters";
 import { FilterCondition } from "../../../types";
-import { isValidTableName } from "../../clickhouse/schema-utils";
+import { isValidTableName } from "../../clickhouse/schemaUtils";
 import { logger } from "../../logger";
 import { UiColumnMapping } from "../../../tableDefinitions";
 import {
@@ -13,6 +13,7 @@ import {
   ArrayOptionsFilter,
   BooleanFilter,
   NumberObjectFilter,
+  StringObjectFilter,
 } from "./clickhouse-filter";
 
 export class QueryBuilderError extends Error {
@@ -79,6 +80,7 @@ export const createFilterFromFilterState = (
           clickhouseTable: column.clickhouseTableName,
           field: column.clickhouseSelect,
           value: frontEndFilter.value,
+          operator: frontEndFilter.operator,
           tablePrefix: column.queryPrefix,
         });
       case "numberObject":
@@ -90,11 +92,19 @@ export const createFilterFromFilterState = (
           value: frontEndFilter.value,
           tablePrefix: column.queryPrefix,
         });
+      case "stringObject":
+        return new StringObjectFilter({
+          clickhouseTable: column.clickhouseTableName,
+          field: column.clickhouseSelect,
+          operator: frontEndFilter.operator,
+          key: frontEndFilter.key,
+          value: frontEndFilter.value,
+          tablePrefix: column.queryPrefix,
+        });
 
       default:
-        throw new QueryBuilderError(
-          `Invalid filter type: ${frontEndFilter.type}`,
-        );
+        logger.error(`Invalid filter type: ${JSON.stringify(frontEndFilter)}`);
+        throw new QueryBuilderError(`Invalid filter type`);
     }
   });
 };
@@ -105,9 +115,9 @@ const matchAndVerifyTracesUiColumn = (
 ) => {
   // tries to match the column name to the clickhouse table name
   logger.debug(`Filter to match: ${JSON.stringify(filter)}`);
-
   const uiTable = uiTableDefinitions.find(
-    (col) => col.uiTableName === filter.column, // matches on the NAME of the column in the UI.
+    (col) =>
+      col.uiTableName === filter.column || col.uiTableId === filter.column, // matches on the NAME of the column in the UI.
   );
 
   if (!uiTable) {

packages/shared/src/server/queries/clickhouse-sql/search.ts

@@ -0,0 +1,20 @@
+const regexIndefiniteCharacters = "%";
+
+export const clickhouseSearchCondition = (query?: string) => {
+  return {
+    query: query
+      ? `
+    AND (
+      id ILIKE {searchString: String} OR 
+      user_id ILIKE {searchString: String} OR 
+      name ILIKE {searchString: String}
+    )
+  `
+      : "",
+    params: query
+      ? {
+          searchString: `${regexIndefiniteCharacters}${query}${regexIndefiniteCharacters}`,
+        }
+      : {},
+  };
+};

packages/shared/src/server/repositories/Readme.md

@@ -0,0 +1,15 @@
+## Repository docs
+
+### Guarantees for relating data within Langfuse
+
+- Finding a Trace based on an observation [Linear](https://linear.app/langfuse/issue/LFE-2745/improve-generations-table-query-performance)
+  - Traces can occur earlier than an observation.
+  - 96% of observations.start_time occur 2 mins later than the trace.timestamp
+  - There is a very large long-tail. Hence we will use a 2-day (2880 min) look back for now.
+- Finding an Observation based on a Trace [Linear](https://linear.app/langfuse/issue/LFE-2409/table-queries)
+  - Observations have a very high likelihood of happening after the trace.
+  - 97% of traces.timestamp occur 2 mins earlier than the observation.start_time
+  - We will maintain a 1 hour cutoff for now.
+- Finding traces/observations based on a Score timestamp
+  - For scores we have a very high likelihood of happening after the trace / observation.
+  - We maintain a 1 hour cutoff for now.

packages/shared/src/server/repositories/clickhouse.ts

@@ -1,48 +1,122 @@
-import { JsonNested } from "../../utils/zod";
 import { env } from "../../env";
-import { clickhouseClient } from "../clickhouse/client";
+import {
+  clickhouseClient,
+  convertDateToClickhouseDateTime,
+} from "../clickhouse/client";
 import { logger } from "../logger";
-import { getCurrentSpan, instrumentAsync } from "../instrumentation";
+import { instrumentAsync } from "../instrumentation";
+import { S3StorageService } from "../services/S3StorageService";
+import { randomUUID } from "crypto";
+import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 
-export const convertRecordToJsonSchema = (
-  record: Record<string, string>,
-): JsonNested | undefined => {
-  const jsonSchema: JsonNested = {};
+let s3StorageServiceClient: S3StorageService;
 
-  // if record is empty, return undefined
-  if (Object.keys(record).length === 0) {
-    return undefined;
+const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
+  if (!s3StorageServiceClient) {
+    s3StorageServiceClient = new S3StorageService({
+      bucketName,
+      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
+      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
+      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
+      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
+      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
+    });
   }
-
-  for (const key in record) {
-    try {
-      jsonSchema[key] = JSON.parse(record[key]);
-    } catch (e) {
-      jsonSchema[key] = record[key];
-    }
-  }
-
-  return jsonSchema;
+  return s3StorageServiceClient;
 };
 
+export async function upsertClickhouse<
+  T extends Record<string, unknown>,
+>(opts: {
+  table: "scores" | "traces"; // TODO: Modify eventType logic to support more tables going forward
+  records: T[];
+  eventBodyMapper: (body: T) => Record<string, unknown>;
+}): Promise<void> {
+  return await instrumentAsync({ name: "clickhouse-upsert" }, async (span) => {
+    // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
+    span.setAttribute("ch.query.table", opts.table);
+
+    // drop trailing s and pretend it's always a create.
+    // Only applicable to scores and traces.
+    const eventType = `${opts.table.slice(0, -1)}-create`;
+
+    // If event upload is enabled, we store all rows in S3 to have a backup
+    if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
+      if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
+        throw new Error("S3 event store is enabled but no bucket is set");
+      }
+      const s3Client = getS3StorageServiceClient(
+        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+      );
+      await Promise.all(
+        opts.records.map((record) => {
+          s3Client.uploadJson(
+            `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
+            [
+              {
+                id: randomUUID(),
+                timestamp: new Date().toISOString(),
+                type: eventType,
+                body: opts.eventBodyMapper(record),
+              },
+            ],
+          );
+        }),
+      );
+    }
+
+    const res = await clickhouseClient.insert({
+      table: opts.table,
+      values: opts.records.map((record) => ({
+        ...record,
+        event_ts: convertDateToClickhouseDateTime(new Date()),
+      })),
+      format: "JSONEachRow",
+    });
+    // same logic as for prisma. we want to see queries in development
+    if (env.NODE_ENV === "development") {
+      logger.info(`clickhouse:insert ${res.query_id} ${opts.table}`);
+    }
+
+    span.setAttribute("ch.queryId", res.query_id);
+
+    // add summary headers to the span. Helps to tune performance
+    const summaryHeader = res.response_headers["x-clickhouse-summary"];
+    if (summaryHeader) {
+      try {
+        const summary = Array.isArray(summaryHeader)
+          ? JSON.parse(summaryHeader[0])
+          : JSON.parse(summaryHeader);
+        for (const key in summary) {
+          span.setAttribute(`ch.${key}`, summary[key]);
+        }
+      } catch (error) {
+        logger.debug(
+          `Failed to parse clickhouse summary header ${summaryHeader}`,
+          error,
+        );
+      }
+    }
+  });
+}
+
 export async function queryClickhouse<T>(opts: {
   query: string;
   params?: Record<string, unknown> | undefined;
-}) {
+}): Promise<T[]> {
   return await instrumentAsync({ name: "clickhouse-query" }, async (span) => {
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.text", opts.query);
 
-    // same logic as for prisma. we want to see queries in development
-    if (env.NODE_ENV === "development") {
-      logger.info(`clickhouse:query ${opts.query}`);
-    }
-
     const res = await clickhouseClient.query({
       query: opts.query,
       format: "JSONEachRow",
       query_params: opts.params,
     });
+    // same logic as for prisma. we want to see queries in development
+    if (env.NODE_ENV === "development") {
+      logger.info(`clickhouse:query ${res.query_id} ${opts.query}`);
+    }
 
     span.setAttribute("ch.queryId", res.query_id);
 
@@ -68,6 +142,45 @@ export async function queryClickhouse<T>(opts: {
   });
 }
 
+export async function commandClickhouse<T>(opts: {
+  query: string;
+  params?: Record<string, unknown> | undefined;
+}): Promise<void> {
+  return await instrumentAsync({ name: "clickhouse-command" }, async (span) => {
+    // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
+    span.setAttribute("ch.query.text", opts.query);
+
+    const res = await clickhouseClient.command({
+      query: opts.query,
+      query_params: opts.params,
+    });
+    // same logic as for prisma. we want to see queries in development
+    if (env.NODE_ENV === "development") {
+      logger.info(`clickhouse:query ${res.query_id} ${opts.query}`);
+    }
+
+    span.setAttribute("ch.queryId", res.query_id);
+
+    // add summary headers to the span. Helps to tune performance
+    const summaryHeader = res.response_headers["x-clickhouse-summary"];
+    if (summaryHeader) {
+      try {
+        const summary = Array.isArray(summaryHeader)
+          ? JSON.parse(summaryHeader[0])
+          : JSON.parse(summaryHeader);
+        for (const key in summary) {
+          span.setAttribute(`ch.${key}`, summary[key]);
+        }
+      } catch (error) {
+        logger.debug(
+          `Failed to parse clickhouse summary header ${summaryHeader}`,
+          error,
+        );
+      }
+    }
+  });
+}
+
 export function parseClickhouseUTCDateTimeFormat(dateStr: string): Date {
   return new Date(`${dateStr.replace(" ", "T")}Z`);
 }

packages/shared/src/server/repositories/constants.ts

@@ -0,0 +1,7 @@
+// t.timestamp > observation.start_time - 2 days
+export const OBSERVATIONS_TO_TRACE_INTERVAL = "INTERVAL 2 DAY";
+// observation.start_time > t.timestamp - 1 hour
+export const TRACE_TO_OBSERVATIONS_INTERVAL = "INTERVAL 1 HOUR";
+// observation.start_time > s.timestamp - 1 hour
+// t.timestamp > s.timestamp - 1 hour
+export const SCORE_TO_TRACE_OBSERVATIONS_INTERVAL = "INTERVAL 1 HOUR";

packages/shared/src/server/repositories/dashboards.ts

@@ -0,0 +1,556 @@
+import { queryClickhouse } from "./clickhouse";
+import { createFilterFromFilterState } from "../queries/clickhouse-sql/factory";
+import { FilterState } from "../../types";
+import {
+  DateTimeFilter,
+  FilterList,
+} from "../queries/clickhouse-sql/clickhouse-filter";
+import { dashboardColumnDefinitions } from "../../tableDefinitions/mapDashboards";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import {
+  SCORE_TO_TRACE_OBSERVATIONS_INTERVAL,
+  TRACE_TO_OBSERVATIONS_INTERVAL,
+} from "./constants";
+
+export type DateTrunc = "year" | "month" | "week" | "day" | "hour" | "minute";
+
+export const getTotalTraces = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  ).apply();
+
+  const query = `
+    SELECT 
+      count(id) as count 
+    FROM traces t FINAL 
+    WHERE project_id = {projectId: String}
+    AND ${chFilter.query}`;
+
+  const result = await queryClickhouse<{ count: number }>({
+    query,
+    params: {
+      projectId,
+      ...chFilter.params,
+    },
+  });
+
+  if (result.length === 0) {
+    return undefined;
+  }
+
+  return [{ countTraceId: result[0].count }];
+};
+
+export const getObservationsCostGroupedByName = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const hasTraceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
+  const query = `
+    SELECT 
+      provided_model_name as name,
+      sumMap(cost_details)['total'] as sum_cost_details,
+      sumMap(usage_details)['total'] as sum_usage_details
+    FROM observations o FINAL ${hasTraceFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+    WHERE project_id = {projectId: String}
+    AND ${appliedFilter.query}
+    GROUP BY provided_model_name
+    ORDER BY sumMap(cost_details)['total'] DESC
+    `;
+
+  const result = await queryClickhouse<{
+    name: string;
+    sum_cost_details: number;
+    sum_usage_details: number;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...appliedFilter.params,
+    },
+  });
+
+  return result;
+};
+
+export const getScoreAggregate = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+
+  const timeFilter = chFilter.find(
+    (f) =>
+      f.field === "timestamp" && (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  const chFilterApplied = chFilter.apply();
+
+  const hasTraceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
+  const query = `
+    SELECT 
+      s.name,
+      count(*) as count,
+      avg(s.value) as avg_value,
+      s.source,
+      s.data_type
+    FROM scores s FINAL 
+     ${hasTraceFilter ? "JOIN traces t FINAL ON t.id = s.trace_id AND t.project_id = s.project_id" : ""}
+    WHERE s.project_id = {projectId: String}
+    AND ${chFilterApplied.query}
+    ${timeFilter && hasTraceFilter ? `AND t.timestamp >= {tracesTimestamp: DateTime} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
+    GROUP BY s.name, s.source, s.data_type
+    ORDER BY count(*) DESC
+    `;
+
+  const result = await queryClickhouse<{
+    name: string;
+    count: string;
+    avg_value: string;
+    source: string;
+    data_type: string;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...chFilterApplied.params,
+      ...(timeFilter
+        ? { tracesTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
+        : {}),
+    },
+  });
+
+  return result;
+};
+
+export const groupTracesByTime = async (
+  projectId: string,
+  filter: FilterState,
+  groupBy: DateTrunc,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  ).apply();
+
+  const query = `
+    SELECT 
+      ${selectTimeseriesColumn(groupBy, "timestamp", "timestamp")},
+      count(*) as count
+    FROM traces t FINAL
+    WHERE project_id = {projectId: String}
+    AND ${chFilter.query}
+    GROUP BY timestamp
+    ${orderByTimeSeries(groupBy, "timestamp")}
+    `;
+
+  const result = await queryClickhouse<{
+    timestamp: string;
+    count: string;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...chFilter.params,
+    },
+  });
+
+  return result.map((row) => ({
+    timestamp: new Date(row.timestamp),
+    countTraceId: Number(row.count),
+  }));
+};
+
+export const getObservationUsageByTime = async (
+  projectId: string,
+  filter: FilterState,
+  groupBy: DateTrunc,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const query = `
+    SELECT 
+      ${selectTimeseriesColumn(groupBy, "start_time", "start_time")},
+      sumMap(usage_details)['total'] as sum_usage_details,
+      sumMap(cost_details)['total'] as sum_cost_details,
+      provided_model_name
+    FROM observations o FINAL
+    ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+    WHERE project_id = {projectId: String}
+    AND ${appliedFilter.query}
+    GROUP BY start_time, provided_model_name
+    ${orderByTimeSeries(groupBy, "start_time")}
+    `;
+
+  const result = await queryClickhouse<{
+    start_time: string;
+    sum_usage_details: string;
+    sum_cost_details: number;
+    provided_model_name: string;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...appliedFilter.params,
+    },
+  });
+
+  return result.map((row) => ({
+    start_time: new Date(row.start_time),
+    sum_usage_details: Number(row.sum_usage_details),
+    sum_cost_details: row.sum_cost_details,
+    provided_model_name: row.provided_model_name,
+  }));
+};
+
+export const getDistinctModels = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const query = `
+    SELECT 
+      distinct(provided_model_name) as model
+    FROM observations o FINAL
+    ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+    WHERE project_id = {projectId: String}
+    AND ${appliedFilter.query}
+    `;
+
+  const result = await queryClickhouse<{ model: string }>({
+    query,
+    params: {
+      projectId,
+      ...appliedFilter.params,
+    },
+  });
+
+  return result;
+};
+
+export const getScoresAggregateOverTime = async (
+  projectId: string,
+  filter: FilterState,
+  groupBy: DateTrunc,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
+  const query = `
+  SELECT 
+    ${selectTimeseriesColumn(groupBy, "timestamp", "timestamp")},
+    name,
+    data_type,
+    source,
+    AVG(value) as avg_value
+  FROM scores FINAL
+  ${traceFilter ? "JOIN traces t ON scores.trace_id = t.id AND scores.project_id = t.project_id" : ""}
+  WHERE project_id = {projectId: String}
+  AND ${appliedFilter.query}
+  AND data_type IN ('NUMERIC', 'BOOLEAN')
+  GROUP BY 
+    timestamp,
+    name,
+    data_type,
+    source
+  ${orderByTimeSeries(groupBy, "timestamp")};
+`;
+
+  const result = await queryClickhouse<{
+    timestamp: string;
+    name: string;
+    data_type: string;
+    source: string;
+    avg_value: number;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...appliedFilter.params,
+    },
+  });
+
+  return result.map((row) => ({
+    scoreTimestamp: new Date(row.timestamp),
+    scoreName: row.name,
+    scoreDataType: row.data_type,
+    scoreSource: row.source,
+    avgValue: Number(row.avg_value),
+  }));
+};
+
+export const getModelUsageByUser = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const timeFilter = chFilter.find(
+    (f) =>
+      f.clickhouseTable === "observations" &&
+      f.field === "start_time" &&
+      (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  const query = `
+    SELECT 
+      sumMap(usage_details)['total'] as sum_usage_details,
+      sumMap(cost_details)['total'] as sum_cost_details,
+      user_id
+    FROM observations o FINAL
+      JOIN traces t FINAL ON o.trace_id = t.id AND o.project_id = t.project_id
+    WHERE project_id = {projectId: String}
+    AND t.user_id IS NOT NULL
+    AND ${appliedFilter.query}
+    ${timeFilter ? `AND t.timestamp >= {tractTimestamp: DateTime} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+    GROUP BY user_id
+    ORDER BY sum_cost_details DESC
+    `;
+
+  const result = await queryClickhouse<{
+    sum_usage_details: string;
+    sum_cost_details: number;
+    user_id: string;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...appliedFilter.params,
+      ...(timeFilter ? { tractTimestamp: timeFilter.value } : {}),
+    },
+  });
+
+  return result.map((row) => ({
+    sumUsageDetails: Number(row.sum_usage_details),
+    sumCostDetails: Number(row.sum_cost_details),
+    userId: row.user_id,
+  }));
+};
+
+export const getObservationLatencies = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const query = `
+    SELECT
+      quantilesExactLow(0.5, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles,
+      name
+    FROM observations o FINAL
+    ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+    WHERE project_id = {projectId: String}
+    AND ${appliedFilter.query}
+    GROUP BY name
+    ORDER BY quantiles[2] DESC
+    `;
+
+  const result = await queryClickhouse<{ quantiles: string[]; name: string }>({
+    query,
+    params: { projectId, ...appliedFilter.params },
+  });
+
+  return result.map((row) => ({
+    p50: Number(row.quantiles[0]) / 1000,
+    p90: Number(row.quantiles[1]) / 1000,
+    p95: Number(row.quantiles[2]) / 1000,
+    p99: Number(row.quantiles[3]) / 1000,
+    name: row.name,
+  }));
+};
+
+export const getTracesLatencies = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const timestampFilter = chFilter.find(
+    (f) =>
+      f.clickhouseTable === "traces" &&
+      f.field === 't."timestamp"' &&
+      (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  const query = `
+    WITH trace_latencies as (
+      select o.trace_id,
+             t.name,
+             o.project_id,
+             date_diff('milliseconds', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
+      FROM traces t FINAL 
+      JOIN observations o FINAL
+      ON o.trace_id = t.id AND o.project_id = t.project_id
+      WHERE project_id = {projectId: String}
+      AND ${appliedFilter.query}
+      ${timestampFilter ? `AND o.start_time > {dateTimeFilterObservations: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+      GROUP BY o.project_id, o.trace_id, t.name
+    )
+
+    SELECT
+      quantilesExactLow(0.5, 0.9, 0.95, 0.99)(duration) as quantiles,
+      name
+    FROM trace_latencies
+    GROUP BY name
+    ORDER BY quantiles[2] DESC
+  `;
+
+  const result = await queryClickhouse<{ quantiles: string[]; name: string }>({
+    query,
+    params: {
+      projectId,
+      ...appliedFilter.params,
+      ...(timestampFilter
+        ? { dateTimeFilterObservations: timestampFilter.value }
+        : {}),
+    },
+  });
+
+  return result.map((row) => ({
+    p50: Number(row.quantiles[0]) / 1000,
+    p90: Number(row.quantiles[1]) / 1000,
+    p95: Number(row.quantiles[2]) / 1000,
+    p99: Number(row.quantiles[3]) / 1000,
+    name: row.name,
+  }));
+};
+
+export const getModelLatenciesOverTime = async (
+  projectId: string,
+  filter: FilterState,
+  groupBy: DateTrunc,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+
+  const appliedFilter = chFilter.apply();
+
+  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
+  const query = `
+  SELECT 
+    ${selectTimeseriesColumn(groupBy, "o.start_time", "start_time_bucket")},
+    provided_model_name,
+    quantilesExactLow(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles
+  FROM observations o FINAL
+  ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+  WHERE project_id = {projectId: String}
+  AND ${appliedFilter.query}
+  GROUP BY provided_model_name, start_time_bucket
+  ${orderByTimeSeries(groupBy, "start_time_bucket")};
+`;
+
+  const result = await queryClickhouse<{
+    start_time_bucket: string;
+    provided_model_name: string;
+    quantiles: string[];
+  }>({ query, params: { projectId, ...appliedFilter.params } });
+
+  return result.map((row) => ({
+    p50: Number(row.quantiles[0]) / 1000,
+    p75: Number(row.quantiles[1]) / 1000,
+    p90: Number(row.quantiles[2]) / 1000,
+    p95: Number(row.quantiles[3]) / 1000,
+    p99: Number(row.quantiles[4]) / 1000,
+    model: row.provided_model_name,
+    start_time: new Date(row.start_time_bucket),
+  }));
+};
+
+const orderByTimeSeries = (dateTrunc: DateTrunc, col: string) => {
+  let interval;
+  switch (dateTrunc) {
+    case "year":
+      interval = "toIntervalYear(1)";
+      break;
+    case "month":
+      interval = "toIntervalMonth(1)";
+      break;
+    case "week":
+      interval = "toIntervalWeek(1)";
+      break;
+    case "day":
+      interval = "toIntervalDay(1)";
+      break;
+    case "hour":
+      interval = "toIntervalHour(1)";
+      break;
+    case "minute":
+      interval = "toIntervalMinute(1)";
+      break;
+    default:
+      return undefined;
+  }
+
+  return `ORDER BY ${col} ASC WITH FILL STEP ${interval}`;
+};
+
+const selectTimeseriesColumn = (
+  dateTrunc: DateTrunc,
+  col: string,
+  as: String,
+) => {
+  let interval;
+  switch (dateTrunc) {
+    case "year":
+      interval = "toStartOfYear";
+      break;
+    case "month":
+      interval = "toStartOfMonth";
+      break;
+    case "week":
+      interval = "toStartOfWeek";
+      break;
+    case "day":
+      interval = "toStartOfDay";
+      break;
+    case "hour":
+      interval = "toStartOfHour";
+      break;
+    case "minute":
+      interval = "toStartOfMinute";
+      break;
+    default:
+      return undefined;
+  }
+  return `${interval}(${col}) as ${as}`;
+};

packages/shared/src/server/repositories/definitions.ts

@@ -9,10 +9,26 @@ export const clickhouseStringDateSchema = z
 
 //https://clickhouse.com/docs/en/integrations/javascript#integral-types-int64-int128-int256-uint64-uint128-uint256
 // clickhouse returns int64 as string
-export const UsageCostStringSchema = z.record(z.string(), z.string());
-export const UsageCostNumberSchema = z.record(z.string(), z.coerce.number());
-export type UsageCostStringType = z.infer<typeof UsageCostStringSchema>;
-export type UsageCostNumberType = z.infer<typeof UsageCostNumberSchema>;
+export const UsageCostSchema = z
+  .record(z.string(), z.coerce.string().nullable())
+  .transform((val, ctx) => {
+    const result: Record<string, number> = {};
+    for (const key in val) {
+      if (val[key] !== null && val[key] !== undefined) {
+        const parsed = Number(val[key]);
+        if (isNaN(parsed)) {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Key ${key} is not a number`,
+          });
+        } else {
+          result[key] = parsed;
+        }
+      }
+    }
+    return result;
+  });
+export type UsageCostType = z.infer<typeof UsageCostSchema>;
 
 export const observationRecordBaseSchema = z.object({
   id: z.string(),
@@ -47,10 +63,10 @@ export const observationRecordReadSchema = observationRecordBaseSchema.extend({
   end_time: clickhouseStringDateSchema.nullish(),
   completion_start_time: clickhouseStringDateSchema.nullish(),
   event_ts: clickhouseStringDateSchema,
-  provided_usage_details: UsageCostStringSchema,
-  provided_cost_details: UsageCostNumberSchema,
-  usage_details: UsageCostStringSchema,
-  cost_details: UsageCostNumberSchema,
+  provided_usage_details: UsageCostSchema,
+  provided_cost_details: UsageCostSchema,
+  usage_details: UsageCostSchema,
+  cost_details: UsageCostSchema,
 });
 export type ObservationRecordReadType = z.infer<
   typeof observationRecordReadSchema
@@ -64,10 +80,10 @@ export const observationRecordInsertSchema = observationRecordBaseSchema.extend(
     end_time: z.number().nullish(),
     completion_start_time: z.number().nullish(),
     event_ts: z.number(),
-    provided_usage_details: UsageCostNumberSchema,
-    provided_cost_details: UsageCostNumberSchema,
-    usage_details: UsageCostNumberSchema,
-    cost_details: UsageCostNumberSchema,
+    provided_usage_details: UsageCostSchema,
+    provided_cost_details: UsageCostSchema,
+    usage_details: UsageCostSchema,
+    cost_details: UsageCostSchema,
   },
 );
 export type ObservationRecordInsertType = z.infer<
@@ -159,11 +175,6 @@ export const convertObservationReadToInsert = (
 ): ObservationRecordInsertType => {
   const convertDate = (date: string) => new Date(date).getTime();
 
-  const convertDetails = (details: Record<string, string>) =>
-    Object.fromEntries(
-      Object.entries(details).map(([key, value]) => [key, Number(value)]),
-    );
-
   return {
     ...record,
     created_at: convertDate(record.created_at),
@@ -174,9 +185,9 @@ export const convertObservationReadToInsert = (
       ? convertDate(record.completion_start_time)
       : undefined,
     event_ts: convertDate(record.event_ts),
-    provided_usage_details: convertDetails(record.provided_usage_details),
+    provided_usage_details: record.provided_usage_details,
     provided_cost_details: record.provided_cost_details,
-    usage_details: convertDetails(record.usage_details),
+    usage_details: record.usage_details,
     cost_details: record.cost_details,
   };
 };
@@ -268,9 +279,12 @@ export const convertPostgresObservationToInsert = (
       : null,
     provided_usage_details: {},
     usage_details: {
-      input: observation.prompt_tokens,
-      output: observation.completion_tokens,
-      total: observation.total_tokens,
+      input: observation.prompt_tokens >= 0 ? observation.prompt_tokens : null,
+      output:
+        observation.completion_tokens >= 0
+          ? observation.completion_tokens
+          : null,
+      total: observation.total_tokens >= 0 ? observation.total_tokens : null,
     },
     provided_cost_details: {
       input: observation.input_cost?.toNumber() ?? null,

packages/shared/src/server/repositories/index.ts

@@ -2,3 +2,6 @@ export * from "./scores";
 export * from "./traces";
 export * from "./observations";
 export * from "./types";
+export * from "./dashboards";
+export * from "./traces_converters";
+export * from "./scores_converters";

packages/shared/src/server/repositories/observations.ts

@@ -1,144 +1,32 @@
-import {
-  parseClickhouseUTCDateTimeFormat,
-  queryClickhouse,
-} from "./clickhouse";
-import {
-  Model,
-  Observation,
-  ObservationLevel,
-  ObservationType,
-  ObservationView,
-  Price,
-} from "@prisma/client";
+import { commandClickhouse, queryClickhouse } from "./clickhouse";
+import { Observation, ObservationLevel } from "@prisma/client";
 import { logger } from "../logger";
 import { InternalServerError, LangfuseNotFoundError } from "../../errors";
-import Decimal from "decimal.js";
 import { prisma } from "../../db";
-import { jsonSchema } from "../../utils/zod";
 import { ObservationRecordReadType } from "./definitions";
 import { FilterState } from "../../types";
 import {
   DateTimeFilter,
   FilterList,
   StringFilter,
-} from "../queries/clickhouse-filter/clickhouse-filter";
+} from "../queries/clickhouse-sql/clickhouse-filter";
 import { FullObservations } from "../queries/createGenerationsQuery";
-import { createFilterFromFilterState } from "../queries/clickhouse-filter/factory";
-import { observationsTableUiColumnDefinitions } from "../../tableDefinitions";
+import { createFilterFromFilterState } from "../queries/clickhouse-sql/factory";
+import {
+  observationsTableTraceUiColumnDefinitions,
+  observationsTableUiColumnDefinitions,
+} from "../../tableDefinitions";
 import { TableCount } from "./types";
-import { orderByToClickhouseSql } from "../queries/clickhouse-filter/orderby-factory";
+import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
 import { OrderByState } from "../../interfaces/orderBy";
-
-export const convertObservation = async (
-  record: ObservationRecordReadType,
-): Promise<Observation> => {
-  const model = record.internal_model_id
-    ? await prisma.model.findFirst({
-        where: {
-          id: record.internal_model_id,
-        },
-        include: {
-          Price: true,
-        },
-      })
-    : undefined;
-  return await convertObservationAndModel(record, model ?? undefined);
-};
-
-export const convertObservationToView = async (
-  record: ObservationRecordReadType,
-): Promise<ObservationView> => {
-  const model = record.internal_model_id
-    ? await prisma.model.findFirst({
-        where: {
-          id: record.internal_model_id,
-        },
-        include: {
-          Price: true,
-        },
-      })
-    : undefined;
-  return {
-    ...(await convertObservationAndModel(record, model ?? undefined)),
-    latency: record.end_time
-      ? parseClickhouseUTCDateTimeFormat(record.end_time).getTime() -
-        parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
-      : null,
-    timeToFirstToken: record.completion_start_time
-      ? parseClickhouseUTCDateTimeFormat(record.start_time).getTime() -
-        parseClickhouseUTCDateTimeFormat(record.completion_start_time).getTime()
-      : null,
-    inputPrice:
-      model?.Price?.find((m) => m.usageType === "input")?.price ?? null,
-    outputPrice:
-      model?.Price?.find((m) => m.usageType === "output")?.price ?? null,
-    totalPrice:
-      model?.Price?.find((m) => m.usageType === "total")?.price ?? null,
-    promptName: record.prompt_name ?? null,
-    promptVersion: record.prompt_version ?? null,
-    modelId: record.internal_model_id ?? null,
-  };
-};
-
-const convertObservationAndModel = async (
-  record: ObservationRecordReadType,
-  model?: Model & { Price: Price[] },
-): Promise<Observation> => {
-  return {
-    id: record.id,
-    traceId: record.trace_id ?? null,
-    projectId: record.project_id,
-    type: record.type as ObservationType,
-    parentObservationId: record.parent_observation_id ?? null,
-    startTime: parseClickhouseUTCDateTimeFormat(record.start_time),
-    endTime: record.end_time
-      ? parseClickhouseUTCDateTimeFormat(record.end_time)
-      : null,
-    name: record.name ?? null,
-    metadata: record.metadata,
-    level: record.level as ObservationLevel,
-    statusMessage: record.status_message ?? null,
-    version: record.version ?? null,
-    input: jsonSchema.nullish().parse(record.input) ?? null,
-    output: jsonSchema.nullish().parse(record.output) ?? null,
-    modelParameters: jsonSchema.nullable().parse(record.model_parameters),
-    completionStartTime: record.completion_start_time
-      ? parseClickhouseUTCDateTimeFormat(record.completion_start_time)
-      : null,
-    promptId: record.prompt_id ?? null,
-    createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
-    updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
-    promptTokens: record.usage_details?.input
-      ? Number(record.usage_details?.input)
-      : 0,
-    completionTokens: record.usage_details?.output
-      ? Number(record.usage_details?.output)
-      : 0,
-    totalTokens: record.usage_details?.total
-      ? Number(record.usage_details?.total)
-      : 0,
-    calculatedInputCost: record.cost_details?.input
-      ? new Decimal(record.cost_details.input)
-      : null,
-    calculatedOutputCost: record.cost_details?.output
-      ? new Decimal(record.cost_details.output)
-      : null,
-    calculatedTotalCost: record.cost_details?.total
-      ? new Decimal(record.cost_details.total)
-      : null,
-    inputCost: record.cost_details?.input
-      ? new Decimal(record.cost_details?.input)
-      : null,
-    outputCost: record.cost_details?.output
-      ? new Decimal(record.cost_details?.output)
-      : null,
-    totalCost: record.total_cost ? new Decimal(record.total_cost) : null,
-    model: record.provided_model_name ?? null,
-    internalModelId: record.internal_model_id ?? null,
-    internalModel: model?.modelName ?? null, // to be removed
-    unit: "TOKENS", // to be removed.
-  };
-};
+import { getTracesByIds } from "./traces";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import {
+  convertObservationToView,
+  convertObservation,
+} from "./observations_converters";
+import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
+import { OBSERVATIONS_TO_TRACE_INTERVAL } from "./constants";
 
 export const getObservationsViewForTrace = async (
   traceId: string,
@@ -250,6 +138,7 @@ export type ObservationTableQuery = {
   projectId: string;
   filter: FilterState;
   orderBy?: OrderByState;
+  searchQuery?: string;
   limit?: number;
   offset?: number;
   selectIOAndMetadata?: boolean;
@@ -287,17 +176,20 @@ export const getObservationsTableCount = async (opts: ObservationTableQuery) =>
 export const getObservationsTable = async (
   opts: ObservationTableQuery,
 ): Promise<FullObservations> => {
-  const records =
-    await getObservationsTableInternal<ObservationsTableQueryResult>({
-      ...opts,
-      select: `o.id as id,
+  const observationRecords = await getObservationsTableInternal<
+    Omit<
+      ObservationsTableQueryResult,
+      "trace_tags" | "trace_name" | "trace_user_id" | "type"
+    >
+  >({
+    ...opts,
+    select: `
+        o.id as id,
         o.name as name,
         o."model_parameters" as model_parameters,
         o.start_time as "start_time",
         o.end_time as "end_time",
         o.trace_id as "trace_id",
-        t.name as "trace_name",
-        t.user_id as "trace_user_id",
         o.completion_start_time as "completion_start_time",
         o.provided_usage_details as "provided_usage_details",
         o.usage_details as "usage_details",
@@ -306,7 +198,6 @@ export const getObservationsTable = async (
         o.level as level,
         o.status_message as "status_message",
         o.version as version,
-        t.tags as "trace_tags",
         o.parent_observation_id as "parent_observation_id",
         o.created_at as "created_at",
         o.updated_at as "updated_at",
@@ -314,20 +205,21 @@ export const getObservationsTable = async (
         o.total_cost as "total_cost",
         internal_model_id as "internal_model_id",
         provided_model_name as "provided_model_name",
-        if(isNull(end_time), NULL, date_diff('seconds', start_time, end_time)) as latency,
-        if(isNull(completion_start_time), NULL,  date_diff('seconds', start_time, completion_start_time)) as "time_to_first_token"`,
-    });
+        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
+        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`,
+  });
+
   const uniqueModels: string[] = Array.from(
     new Set(
-      records
+      observationRecords
         .map((r) => r.internal_model_id)
         .filter((r): r is string => Boolean(r)),
     ),
   );
 
-  const models =
+  const [models, traces] = await Promise.all([
     uniqueModels.length > 0
-      ? await prisma.model.findMany({
+      ? prisma.model.findMany({
           where: {
             id: {
               in: uniqueModels,
@@ -338,64 +230,34 @@ export const getObservationsTable = async (
             Price: true,
           },
         })
-      : [];
+      : [],
+    getTracesByIds(
+      observationRecords
+        .map((o) => o.trace_id)
+        .filter((o): o is string => Boolean(o)),
+      opts.projectId,
+    ),
+  ]);
 
-  return records.map((o) => {
-    const model = models.find((p) => p.id === o.internal_model_id);
-    return {
-      id: o.id,
-      type: "GENERATION",
-      name: o.name ?? null,
-      level: o.level as ObservationLevel,
-      version: o.version ?? null,
-      input: o.input ?? null,
-      output: o.output ?? null,
-      metadata: o.metadata,
-      traceId: o.trace_id ?? null,
-      projectId: o.project_id,
-      startTime: parseClickhouseUTCDateTimeFormat(o.start_time),
-      endTime: o.end_time ? parseClickhouseUTCDateTimeFormat(o.end_time) : null,
-      parentObservationId: o.parent_observation_id ?? null,
-      statusMessage: o.status_message ?? null,
-      createdAt: parseClickhouseUTCDateTimeFormat(o.created_at),
-      updatedAt: parseClickhouseUTCDateTimeFormat(o.updated_at),
-      model: o.provided_model_name ?? null,
-      modelParameters: o.model_parameters ?? null,
-      promptTokens: o.usage_details?.input ? Number(o.usage_details?.input) : 0,
-      completionTokens: o.usage_details?.output
-        ? Number(o.usage_details?.output)
-        : 0,
-      totalTokens: o.usage_details?.total ? Number(o.usage_details?.total) : 0,
-      unit: "TOKENS",
-      calculatedInputCost: o.cost_details?.input
-        ? new Decimal(o.cost_details.input)
-        : null,
-      calculatedOutputCost: o.cost_details?.output
-        ? new Decimal(o.cost_details.output)
-        : null,
-      calculatedTotalCost: o.total_cost ? new Decimal(o.total_cost) : null,
-      completionStartTime: o.completion_start_time
-        ? parseClickhouseUTCDateTimeFormat(o.completion_start_time)
-        : null,
-      latency: o.latency ? Number(o.latency) : null,
-      timeToFirstToken: o.time_to_first_token
-        ? Number(o.time_to_first_token)
-        : null,
-      promptId: o.prompt_id ?? null,
-      promptName: o.prompt_name ?? null,
-      promptVersion: o.prompt_version ?? null,
-      modelId: o.internal_model_id ?? null,
-      inputPrice:
-        model?.Price?.find((m) => m.usageType === "input")?.price ?? null,
-      outputPrice:
-        model?.Price?.find((m) => m.usageType === "output")?.price ?? null,
-      totalPrice:
-        model?.Price?.find((m) => m.usageType === "total")?.price ?? null,
-      traceName: o.trace_name ?? null,
-      traceTags: o.trace_tags ?? [],
-      userId: o.trace_user_id ?? null,
-    };
-  });
+  return await Promise.all(
+    observationRecords.map(async (o) => {
+      const model = models.find((p) => p.id === o.internal_model_id);
+      const trace = traces.find((t) => t.id === o.trace_id);
+      return {
+        ...(await convertObservationToView(
+          { ...o, type: "GENERATION" },
+          model,
+        )),
+        latency: o.latency ? Number(o.latency) / 1000 : null,
+        timeToFirstToken: o.time_to_first_token
+          ? Number(o.time_to_first_token) / 1000
+          : null,
+        traceName: trace?.name ?? null,
+        traceTags: trace?.tags ?? [],
+        userId: trace?.userId ?? null,
+      };
+    }),
+  );
 };
 
 const getObservationsTableInternal = async <T>(
@@ -424,6 +286,18 @@ const getObservationsTableInternal = async <T>(
     (f) =>
       f.column === "Start Time" && (f.operator === ">=" || f.operator === ">"),
   );
+
+  // query optimisation: joining traces onto observations is expensive. Hence, only join if the UI table contains filters on traces.
+  const traceTableFilter = opts.filter.filter(
+    (f) =>
+      observationsTableTraceUiColumnDefinitions
+        .map((c) => c.uiTableId)
+        .includes(f.column) ||
+      observationsTableTraceUiColumnDefinitions
+        .map((c) => c.uiTableName)
+        .includes(f.column),
+  );
+
   timeFilter
     ? scoresFilter.push(
         new DateTimeFilter({
@@ -455,51 +329,93 @@ const getObservationsTableInternal = async <T>(
   const appliedScoresFilter = scoresFilter.apply();
   const appliedObservationsFilter = observationsFilter.apply();
 
-  const query = `
-      WITH scores_avg AS (
-        SELECT
-          trace_id,
-          observation_id,
-           groupArray(tuple(name, avg_value)) AS "scores_avg"
-        FROM (
-          SELECT
-            trace_id,
-            observation_id,
-            name,
-            avg(value) avg_value,
-            comment
-          FROM
-            scores final
-          WHERE ${appliedScoresFilter.query}
-          GROUP BY
-            trace_id,
-            observation_id,
-            name,
-            comment
-          ORDER BY
-            trace_id
-          ) tmp
-        GROUP BY
-          trace_id, 
-          observation_id
-      )
+  const search = clickhouseSearchCondition(opts.searchQuery);
+
+  const scoresCte = `WITH scores_avg AS (
+    SELECT
+      trace_id,
+      observation_id,
+       groupArray(tuple(name, avg_value)) AS "scores_avg"
+    FROM (
+      SELECT
+        trace_id,
+        observation_id,
+        name,
+        avg(value) avg_value,
+        comment
+      FROM
+        scores final
+      WHERE ${appliedScoresFilter.query}
+      GROUP BY
+        trace_id,
+        observation_id,
+        name,
+        comment
+      ORDER BY
+        trace_id
+      ) tmp
+    GROUP BY
+      trace_id, 
+      observation_id
+  )`;
+
+  if (traceTableFilter.length > 0) {
+    // joins with traces are very expensive. We need to filter by time as well.
+    // We assume that a trace has to have been within the last 2 days to be relevant.
+
+    const query = `
+      ${scoresCte}
       SELECT
        ${selectString}
       FROM observations o FINAL 
         LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id
-        LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = t.id and s_avg.observation_id = o.id
+        LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id
+      WHERE ${appliedObservationsFilter.query}
+        AND o.type = 'GENERATION'
+        ${timeFilter ? `AND t.timestamp > {tracesTimestampFilter: DateTime} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        ${search.query}
+      ${orderByToClickhouseSql(orderBy ?? null, observationsTableUiColumnDefinitions)}
+      ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
+
+    const res = await queryClickhouse<T>({
+      query,
+      params: {
+        ...appliedScoresFilter.params,
+        ...appliedObservationsFilter.params,
+        ...(timeFilter
+          ? {
+              tracesTimestampFilter: convertDateToClickhouseDateTime(
+                timeFilter.value as Date,
+              ),
+            }
+          : {}),
+        ...search.params,
+      },
+    });
+
+    return res;
+  } else {
+    // we query by T, which could also be {count: string}.
+    const query = `
+      ${scoresCte}
+      SELECT
+       ${selectString}
+      FROM observations o FINAL 
+        LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id
       WHERE ${appliedObservationsFilter.query}
       ${orderByToClickhouseSql(orderBy ?? null, observationsTableUiColumnDefinitions)}
       ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
 
-  const res = await queryClickhouse<T>({
-    query,
-    params: {
-      ...appliedScoresFilter.params,
-      ...appliedObservationsFilter.params,
-    },
-  });
-  return res;
+    const res = await queryClickhouse<T>({
+      query,
+      params: {
+        ...appliedScoresFilter.params,
+        ...appliedObservationsFilter.params,
+      },
+    });
+
+    return res;
+  }
 };
 
 export const getObservationsGroupedByModel = async (
@@ -531,6 +447,7 @@ export const getObservationsGroupedByModel = async (
       o.provided_model_name as name
     FROM observations o FINAL
     WHERE ${appliedObservationsFilter.query}
+    AND o.type = 'GENERATION'
     GROUP BY o.provided_model_name
     ORDER BY count() DESC
     LIMIT 1000;
@@ -574,6 +491,7 @@ export const getObservationsGroupedByName = async (
       o.name as name
     FROM observations o FINAL
     WHERE ${appliedObservationsFilter.query}
+    AND o.type = 'GENERATION'
     GROUP BY o.name
     ORDER BY count() DESC
     LIMIT 1000;
@@ -652,3 +570,43 @@ export const getObservationsGroupedByPromptName = async (
     promptName: p.name,
   }));
 };
+
+export const getCostForTraces = async (
+  projectId: string,
+  traceIds: string[],
+) => {
+  const query = `
+    SELECT
+      sum(o.total_cost) as total_cost
+    FROM observations o FINAL
+    WHERE o.project_id = {projectId: String}
+    AND o.trace_id IN ({traceIds: Array(String)});
+    `;
+
+  const res = await queryClickhouse<{ total_cost: string }>({
+    query,
+    params: {
+      projectId,
+      traceIds,
+    },
+  });
+  return res.length > 0 ? Number(res[0].total_cost) : undefined;
+};
+
+export const deleteObservationsByTraceIds = async (
+  projectId: string,
+  traceIds: string[],
+) => {
+  const query = `
+    DELETE FROM observations
+    WHERE project_id = {projectId: String}
+    AND trace_id IN ({traceIds: Array(String)});
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      traceIds,
+    },
+  });
+};

packages/shared/src/server/repositories/observations_converters.ts

@@ -0,0 +1,129 @@
+import {
+  Observation,
+  ObservationView,
+  Model,
+  Price,
+  ObservationType,
+  ObservationLevel,
+} from "@prisma/client";
+import Decimal from "decimal.js";
+import { prisma } from "../../db";
+import { jsonSchema } from "../../utils/zod";
+import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
+import { ObservationRecordReadType } from "./definitions";
+
+export const convertObservation = async (
+  record: ObservationRecordReadType,
+): Promise<Observation> => {
+  const model = record.internal_model_id
+    ? await prisma.model.findFirst({
+        where: {
+          id: record.internal_model_id,
+        },
+        include: {
+          Price: true,
+        },
+      })
+    : undefined;
+  return convertObservationAndModel(record, model ?? undefined);
+};
+
+export const convertObservationToView = async (
+  record: ObservationRecordReadType,
+  providedModel?: Model & { Price: Price[] },
+): Promise<ObservationView> => {
+  const model =
+    providedModel ??
+    (record.internal_model_id
+      ? await prisma.model.findFirst({
+          where: {
+            id: record.internal_model_id,
+          },
+          include: {
+            Price: true,
+          },
+        })
+      : undefined);
+  return {
+    ...convertObservationAndModel(record, model ?? undefined),
+    latency: record.end_time
+      ? parseClickhouseUTCDateTimeFormat(record.end_time).getTime() -
+        parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
+      : null,
+    timeToFirstToken: record.completion_start_time
+      ? parseClickhouseUTCDateTimeFormat(record.start_time).getTime() -
+        parseClickhouseUTCDateTimeFormat(record.completion_start_time).getTime()
+      : null,
+    inputPrice:
+      model?.Price?.find((m) => m.usageType === "input")?.price ?? null,
+    outputPrice:
+      model?.Price?.find((m) => m.usageType === "output")?.price ?? null,
+    totalPrice:
+      model?.Price?.find((m) => m.usageType === "total")?.price ?? null,
+    promptName: record.prompt_name ?? null,
+    promptVersion: record.prompt_version ?? null,
+    modelId: record.internal_model_id ?? null,
+  };
+};
+
+export const convertObservationAndModel = (
+  record: ObservationRecordReadType,
+  model?: Model & { Price: Price[] },
+): Observation => {
+  return {
+    id: record.id,
+    traceId: record.trace_id ?? null,
+    projectId: record.project_id,
+    type: record.type as ObservationType,
+    parentObservationId: record.parent_observation_id ?? null,
+    startTime: parseClickhouseUTCDateTimeFormat(record.start_time),
+    endTime: record.end_time
+      ? parseClickhouseUTCDateTimeFormat(record.end_time)
+      : null,
+    name: record.name ?? null,
+    metadata: record.metadata,
+    level: record.level as ObservationLevel,
+    statusMessage: record.status_message ?? null,
+    version: record.version ?? null,
+    input: jsonSchema.nullish().parse(record.input) ?? null,
+    output: jsonSchema.nullish().parse(record.output) ?? null,
+    modelParameters: record.model_parameters
+      ? JSON.parse(record.model_parameters)
+      : null,
+    completionStartTime: record.completion_start_time
+      ? parseClickhouseUTCDateTimeFormat(record.completion_start_time)
+      : null,
+    promptId: record.prompt_id ?? null,
+    createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
+    updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
+    promptTokens: record.usage_details?.input
+      ? Number(record.usage_details?.input)
+      : 0,
+    completionTokens: record.usage_details?.output
+      ? Number(record.usage_details?.output)
+      : 0,
+    totalTokens: record.usage_details?.total
+      ? Number(record.usage_details?.total)
+      : 0,
+    calculatedInputCost: record.cost_details?.input
+      ? new Decimal(record.cost_details.input)
+      : null,
+    calculatedOutputCost: record.cost_details?.output
+      ? new Decimal(record.cost_details.output)
+      : null,
+    calculatedTotalCost: record.cost_details?.total
+      ? new Decimal(record.cost_details.total)
+      : null,
+    inputCost: record.cost_details?.input
+      ? new Decimal(record.cost_details?.input)
+      : null,
+    outputCost: record.cost_details?.output
+      ? new Decimal(record.cost_details?.output)
+      : null,
+    totalCost: record.total_cost ? new Decimal(record.total_cost) : null,
+    model: record.provided_model_name ?? null,
+    internalModelId: record.internal_model_id ?? null,
+    internalModel: model?.modelName ?? null, // to be removed
+    unit: "TOKENS", // to be removed.
+  };
+};

packages/shared/src/server/repositories/scores.ts

@@ -1,8 +1,20 @@
-import { ScoreDataType, ScoreSource } from "@prisma/client";
-import { queryClickhouse } from "./clickhouse";
-import { FilterList } from "../queries/clickhouse-filter/clickhouse-filter";
+import { Score, ScoreDataType, ScoreSource } from "@prisma/client";
+import {
+  commandClickhouse,
+  parseClickhouseUTCDateTimeFormat,
+  queryClickhouse,
+  upsertClickhouse,
+} from "./clickhouse";
+import { FilterList } from "../queries/clickhouse-sql/clickhouse-filter";
 import { FilterState } from "../../types";
-import { createFilterFromFilterState } from "../queries/clickhouse-filter/factory";
+import {
+  createFilterFromFilterState,
+  getProjectIdDefaultFilter,
+} from "../queries/clickhouse-sql/factory";
+import { OrderByState } from "../../interfaces/orderBy";
+import { scoresTableUiColumnDefinitions } from "../../tableDefinitions";
+import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
+import { convertToScore } from "./scores_converters";
 
 export type FetchScoresReturnType = {
   id: string;
@@ -26,25 +38,84 @@ export type FetchScoresReturnType = {
   projectId: string;
 };
 
-const convertToScore = (row: FetchScoresReturnType) => {
-  return {
-    id: row.id,
-    timestamp: new Date(row.timestamp),
-    projectId: row.project_id,
-    traceId: row.trace_id,
-    observationId: row.observation_id,
-    name: row.name,
-    value: row.value,
-    source: row.source as ScoreSource,
-    comment: row.comment,
-    authorUserId: row.author_user_id,
-    configId: row.config_id,
-    dataType: row.data_type as ScoreDataType,
-    stringValue: row.string_value,
-    queueId: row.queue_id,
-    createdAt: new Date(row.created_at),
-    updatedAt: new Date(row.updated_at),
-  };
+export const searchExistingAnnotationScore = async (
+  projectId: string,
+  traceId: string,
+  observationId: string | null,
+  name: string | undefined,
+  configId: string | undefined,
+) => {
+  if (!name && !configId) {
+    throw new Error("Either name or configId (or both) must be provided.");
+  }
+  const query = `
+    SELECT *
+    FROM scores s FINAL
+    WHERE s.project_id = {projectId: String}
+    AND s.source = 'ANNOTATION'
+    AND s.trace_id = {traceId: String}
+    ${observationId ? `AND s.observation_id = {observationId: String}` : "AND isNull(s.observation_id)"}
+    AND (
+      FALSE
+      ${name ? `OR s.name = {name: String}` : ""}
+      ${configId ? `OR s.config_id = {configId: String}` : ""}
+    )
+    ORDER BY s.event_ts DESC
+    LIMIT 1
+  `;
+
+  const rows = await queryClickhouse<FetchScoresReturnType>({
+    query,
+    params: {
+      projectId,
+      name,
+      configId,
+      traceId,
+      observationId,
+    },
+  });
+  return rows.map(convertToScore).shift();
+};
+
+export const getScoreById = async (
+  projectId: string,
+  scoreId: string,
+  source: ScoreSource,
+) => {
+  const query = `
+    SELECT *
+    FROM scores s FINAL
+    WHERE s.project_id = {projectId: String}
+    AND s.id = {scoreId: String}
+    AND s.source = {source: String}
+    ORDER BY s.event_ts DESC
+    LIMIT 1
+  `;
+
+  const rows = await queryClickhouse<FetchScoresReturnType>({
+    query,
+    params: {
+      projectId,
+      scoreId,
+      source,
+    },
+  });
+  return rows.map(convertToScore).shift();
+};
+
+/**
+ * Accepts a score in a Clickhouse-ready format.
+ * id, project_id, name, and timestamp must always be provided.
+ */
+export const upsertScore = async (score: Partial<FetchScoresReturnType>) => {
+  if (!["id", "project_id", "name", "timestamp"].every((key) => key in score)) {
+    throw new Error("Identifier fields must be provided to upsert Score.");
+  }
+  await upsertClickhouse({
+    table: "scores",
+    records: [score as FetchScoresReturnType],
+    eventBodyMapper: convertToScore,
+  });
 };
 
 export const getScoresForTraces = async (
@@ -177,3 +248,226 @@ export const getScoresGroupedByName = async (
 
   return rows;
 };
+
+export const getScoresUiCount = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy: OrderByState;
+  limit?: number;
+  offset?: number;
+}) => {
+  const rows = await getScoresUiGeneric<{ count: string }>({
+    select: `
+    count(*) as count
+    `,
+    ...props,
+  });
+
+  return Number(rows[0].count);
+};
+
+export type ScoreUiTableRow = Score & {
+  traceName: string | null;
+  traceUserId: string | null;
+  traceTags: Array<string> | null;
+};
+
+export const getScoresUiTable = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy: OrderByState;
+  limit?: number;
+  offset?: number;
+}): Promise<ScoreUiTableRow[]> => {
+  const rows = await getScoresUiGeneric<{
+    id: string;
+    project_id: string;
+    name: string;
+    value: number;
+    string_value: string | null;
+    timestamp: string;
+    source: string;
+    data_type: string;
+    comment: string | null;
+    trace_id: string;
+    observation_id: string | null;
+    author_user_id: string | null;
+    user_id: string | null;
+    trace_name: string | null;
+    trace_tags: Array<string> | null;
+    job_configuration_id: string | null;
+    author_user_image: string | null;
+    author_user_name: string | null;
+    config_id: string | null;
+    queue_id: string | null;
+    created_at: string;
+    updated_at: string;
+  }>({
+    select: `
+        s.id,
+        s.project_id,
+        s.name,
+        s.value,
+        s.string_value,
+        s.timestamp,
+        s.source,
+        s.data_type,
+        s.comment,
+        s.trace_id,
+        s.observation_id,
+        s.author_user_id,
+        t.user_id,
+        t.name,
+        t.tags,
+        s.created_at,
+        s.updated_at,
+        s.source,
+        s.config_id,
+        s.queue_id,
+        t.user_id,
+        t.name as trace_name,
+        t.tags as trace_tags
+    `,
+    ...props,
+  });
+
+  return rows.map((row) => ({
+    projectId: row.project_id,
+    authorUserId: row.author_user_id,
+    traceId: row.trace_id,
+    observationId: row.observation_id,
+    traceUserId: row.user_id,
+    traceName: row.trace_name,
+    traceTags: row.trace_tags,
+    configId: row.config_id,
+    queueId: row.queue_id,
+    createdAt: parseClickhouseUTCDateTimeFormat(row.created_at),
+    updatedAt: parseClickhouseUTCDateTimeFormat(row.updated_at),
+    stringValue: row.string_value,
+    comment: row.comment,
+    dataType: row.data_type as ScoreDataType,
+    source: row.source as ScoreSource,
+    name: row.name,
+    value: row.value,
+    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
+    id: row.id,
+  }));
+};
+
+export const getScoresUiGeneric = async <T>(props: {
+  select: string;
+  projectId: string;
+  filter: FilterState;
+  orderBy: OrderByState;
+  limit?: number;
+  offset?: number;
+}): Promise<T[]> => {
+  const { select, projectId, filter, orderBy, limit, offset } = props;
+
+  const { tracesFilter, scoresFilter, observationsFilter } =
+    getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
+
+  scoresFilter.push(
+    ...createFilterFromFilterState(filter, scoresTableUiColumnDefinitions),
+  );
+
+  const scoresFilterRes = scoresFilter.apply();
+
+  const query = `
+      SELECT 
+          ${select}
+      FROM scores s final
+      LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id
+      WHERE s.project_id = {projectId: String}
+      ${scoresFilterRes?.query ? `AND ${scoresFilterRes.query}` : ""}
+      ${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitions)}
+      ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
+
+    `;
+
+  const rows = await queryClickhouse<T>({
+    query: query,
+    params: {
+      projectId: projectId,
+      ...(scoresFilterRes ? scoresFilterRes.params : {}),
+      limit: limit,
+      offset: offset,
+    },
+  });
+
+  return rows;
+};
+
+export const getScoreNames = async (
+  projectId: string,
+  timestampFilter: FilterState,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(
+      timestampFilter,
+      scoresTableUiColumnDefinitions,
+    ),
+  );
+  const timestampFilterRes = chFilter.apply();
+
+  const query = `
+      select 
+        name,
+        count(*) as count
+      from scores s final
+      WHERE s.project_id = {projectId: String}
+      ${timestampFilterRes?.query ? `AND ${timestampFilterRes.query}` : ""}
+      GROUP BY name
+      ORDER BY count() desc
+      LIMIT 1000;
+    `;
+
+  const rows = await queryClickhouse<{
+    name: string;
+    count: string;
+  }>({
+    query: query,
+    params: {
+      projectId: projectId,
+      ...(timestampFilterRes ? timestampFilterRes.params : {}),
+    },
+  });
+
+  return rows.map((row) => ({
+    name: row.name,
+    count: Number(row.count),
+  }));
+};
+
+export const deleteScore = async (projectId: string, scoreId: string) => {
+  const query = `
+    DELETE FROM scores
+    WHERE project_id = {projectId: String}
+    AND id = {scoreId: String};
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      scoreId,
+    },
+  });
+};
+
+export const deleteScoresByTraceIds = async (
+  projectId: string,
+  traceIds: string[],
+) => {
+  const query = `
+    DELETE FROM scores
+    WHERE project_id = {projectId: String}
+    AND trace_id IN ({traceIds: Array(String)});
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      traceIds,
+    },
+  });
+};

packages/shared/src/server/repositories/scores_converters.ts

@@ -0,0 +1,23 @@
+import { ScoreSource, ScoreDataType } from "@prisma/client";
+import { FetchScoresReturnType } from "./scores";
+
+export const convertToScore = (row: FetchScoresReturnType) => {
+  return {
+    id: row.id,
+    timestamp: new Date(row.timestamp),
+    projectId: row.project_id,
+    traceId: row.trace_id,
+    observationId: row.observation_id,
+    name: row.name,
+    value: row.value,
+    source: row.source as ScoreSource,
+    comment: row.comment,
+    authorUserId: row.author_user_id,
+    configId: row.config_id,
+    dataType: row.data_type as ScoreDataType,
+    stringValue: row.string_value,
+    queueId: row.queue_id,
+    createdAt: new Date(row.created_at),
+    updatedAt: new Date(row.updated_at),
+  };
+};

packages/shared/src/server/repositories/traces.ts

@@ -1,42 +1,32 @@
 import {
+  commandClickhouse,
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
+  upsertClickhouse,
 } from "./clickhouse";
 import {
   createFilterFromFilterState,
   getProjectIdDefaultFilter,
-} from "../queries/clickhouse-filter/factory";
+} from "../queries/clickhouse-sql/factory";
 import { ObservationLevel, Trace } from "@prisma/client";
 import { FilterState } from "../../types";
 import { logger } from "../logger";
-import { FilterList } from "../queries/clickhouse-filter/clickhouse-filter";
+import {
+  DateTimeFilter,
+  FilterList,
+  StringFilter,
+  StringOptionsFilter,
+} from "../queries/clickhouse-sql/clickhouse-filter";
 import { TraceRecordReadType } from "./definitions";
 import { tracesTableUiColumnDefinitions } from "../../tableDefinitions/mapTracesTable";
-import { TableCount } from "./types";
 import { OrderByState } from "../../interfaces/orderBy";
-import { orderByToClickhouseSql } from "../queries/clickhouse-filter/orderby-factory";
-
-const convertClickhouseToDomain = (record: TraceRecordReadType): Trace => {
-  return {
-    id: record.id,
-    projectId: record.project_id,
-    name: record.name ?? null,
-    timestamp: parseClickhouseUTCDateTimeFormat(record.timestamp),
-    tags: record.tags,
-    bookmarked: record.bookmarked,
-    release: record.release ?? null,
-    version: record.version ?? null,
-    userId: record.user_id ?? null,
-    sessionId: record.session_id ?? null,
-    public: record.public,
-    input: record.input ?? null,
-    output: record.output ?? null,
-    metadata: record.metadata,
-    createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
-    updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
-    externalId: null,
-  };
-};
+import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
+import { UiColumnMapping } from "../../tableDefinitions";
+import { sessionCols } from "../../tableDefinitions/mapSessionTable";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { convertClickhouseToDomain } from "./traces_converters";
+import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
+import { TRACE_TO_OBSERVATIONS_INTERVAL } from "./constants";
 
 export type TracesTableReturnType = Pick<
   TraceRecordReadType,
@@ -55,31 +45,36 @@ export type TracesTableReturnType = Pick<
 > & {
   level: ObservationLevel;
   observation_count: number | null;
-  latency: string | null;
+  latency_milliseconds: string | null;
   usage_details: Record<string, number>;
   cost_details: Record<string, number>;
   scores_avg: Array<{ name: string; avg_value: number }>;
 };
 
-export const getTracesTableCount = async (
-  projectId: string,
-  filter: FilterState,
-  orderBy?: OrderByState,
-  limit?: number,
-  offset?: number,
-) =>
-  getTracesTableGeneric<TableCount>({
+export const getTracesTableCount = async (props: {
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  offset?: number;
+}) => {
+  const countRows = await getTracesTableGeneric<{ count: string }>({
     select: "count(*) as count",
-    projectId,
-    filter,
-    orderBy,
-    limit,
-    offset,
+    ...props,
   });
 
+  const converted = countRows.map((row) => ({
+    count: Number(row.count),
+  }));
+
+  return converted.length > 0 ? converted[0].count : 0;
+};
+
 export const getTracesTable = async (
   projectId: string,
   filter: FilterState,
+  searchQuery?: string,
   orderBy?: OrderByState,
   limit?: number,
   offset?: number,
@@ -96,7 +91,7 @@ export const getTracesTable = async (
     t.version, 
     t.user_id, 
     t.session_id,
-    os.latencyMs as latency,
+    os.latency_milliseconds,
     os.cost_details as cost_details,
     os.usage_details as usage_details,
     os.level as level,
@@ -106,6 +101,7 @@ export const getTracesTable = async (
     t.public`,
     projectId,
     filter,
+    searchQuery,
     orderBy,
     limit,
     offset,
@@ -118,14 +114,16 @@ type FetchTracesTableProps = {
   select: string;
   projectId: string;
   filter: FilterState;
+  searchQuery?: string;
   orderBy?: OrderByState;
   limit?: number;
   offset?: number;
 };
 
 const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
-  const { select, projectId, filter, orderBy, limit, offset } = props;
-  logger.info(`input filter ${JSON.stringify(filter)}`);
+  const { select, projectId, filter, orderBy, limit, offset, searchQuery } =
+    props;
+
   const { tracesFilter, scoresFilter, observationsFilter } =
     getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
 
@@ -133,9 +131,71 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
     ...createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
   );
 
+  const traceIdFilter = tracesFilter.find(
+    (f) => f.clickhouseTable === "traces" && f.field === "id",
+  ) as StringFilter | StringOptionsFilter | undefined;
+
+  traceIdFilter
+    ? scoresFilter.push(
+        new StringOptionsFilter({
+          clickhouseTable: "scores",
+          field: "trace_id",
+          operator: "any of",
+          values:
+            traceIdFilter instanceof StringFilter
+              ? [traceIdFilter.value]
+              : traceIdFilter.values,
+        }),
+      )
+    : null;
+  traceIdFilter
+    ? observationsFilter.push(
+        new StringOptionsFilter({
+          clickhouseTable: "observations",
+          field: "trace_id",
+          operator: "any of",
+          values:
+            traceIdFilter instanceof StringFilter
+              ? [traceIdFilter.value]
+              : traceIdFilter.values,
+        }),
+      )
+    : null;
+
+  // for query optimisation, we have to add the timeseries filter to observations + scores as well
+  // stats show, that 98% of all observations have their start_time larger than trace.timestamp - 5 min
+  const timeStampFilter = tracesFilter.find(
+    (f) =>
+      f.field === "timestamp" && (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  timeStampFilter
+    ? scoresFilter.push(
+        new DateTimeFilter({
+          clickhouseTable: "scores",
+          field: "timestamp",
+          operator: ">=",
+          value: timeStampFilter.value,
+        }),
+      )
+    : null;
+
+  timeStampFilter
+    ? observationsFilter.push(
+        new DateTimeFilter({
+          clickhouseTable: "observations",
+          field: "start_time",
+          operator: ">=",
+          value: timeStampFilter.value,
+        }),
+      )
+    : null;
+
   const tracesFilterRes = tracesFilter.apply();
-  const scoresAvgFilterRes = scoresFilter.apply();
-  const observationsStatsRes = observationsFilter.apply();
+  const scoresFilterRes = scoresFilter.apply();
+  const observationFilterRes = observationsFilter.apply();
+
+  const search = clickhouseSearchCondition(searchQuery);
 
   const query = `
   WITH observations_stats AS (
@@ -143,7 +203,7 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
     COUNT(*) AS observation_count,
       sumMap(usage_details) as usage_details,
       SUM(total_cost) AS total_cost,
-      date_diff('seconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latencyMs,
+      date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
       multiIf(
         arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
         arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
@@ -155,7 +215,8 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
       project_id
     FROM
         observations final
-    WHERE ${observationsStatsRes.query}
+    WHERE ${observationFilterRes.query}
+    
     group by trace_id, project_id
 ),
 
@@ -168,7 +229,7 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
                                           name,
                                           avg(value) avg_value
                                   FROM scores final
-                                  WHERE ${scoresAvgFilterRes.query}
+                                  WHERE ${scoresFilterRes.query}
                                   GROUP BY project_id,
                                             trace_id,
                                             name
@@ -182,36 +243,102 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
               left join scores_avg s on s.project_id = t.project_id and s.trace_id = t.id
 
       WHERE ${tracesFilterRes.query}
+      ${search.query}
       ${orderByToClickhouseSql(orderBy ?? null, tracesTableUiColumnDefinitions)}
       ${limit !== undefined && offset !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
     `;
 
-  return await queryClickhouse<T>({
+  const res = await queryClickhouse<T>({
     query: query,
     params: {
       limit: limit,
       offset: offset,
       ...tracesFilterRes.params,
-      ...observationsStatsRes.params,
-      ...scoresAvgFilterRes.params,
+      ...observationFilterRes.params,
+      ...scoresFilterRes.params,
+      ...search.params,
     },
   });
+
+  return res;
+};
+
+/**
+ * Accepts a trace in a Clickhouse-ready format.
+ * id, project_id, and timestamp must always be provided.
+ */
+export const upsertTrace = async (trace: Partial<TraceRecordReadType>) => {
+  if (!["id", "project_id", "timestamp"].every((key) => key in trace)) {
+    throw new Error("Identifier fields must be provided to upsert Trace.");
+  }
+  await upsertClickhouse({
+    table: "traces",
+    records: [trace as TraceRecordReadType],
+    eventBodyMapper: convertClickhouseToDomain,
+  });
+};
+
+export const getTraceById = async (
+  traceId: string,
+  projectId: string,
+  timestamp?: Date,
+): Promise<Trace | undefined> => {
+  try {
+    return getTraceByIdOrThrow(traceId, projectId, timestamp);
+  } catch (e) {
+    return undefined;
+  }
+};
+
+export const getTracesByIds = async (
+  traceIds: string[],
+  projectId: string,
+  timestamp?: Date,
+) => {
+  const query = `
+      SELECT * 
+      FROM traces
+      WHERE id IN ({traceIds: Array(String)})
+        AND project_id = {projectId: String}
+        ${timestamp ? `AND timestamp >= {timestamp: DateTime}` : ""} 
+      ORDER BY event_ts DESC LIMIT 1 by id, project_id;`;
+  const records = await queryClickhouse<TraceRecordReadType>({
+    query,
+    params: {
+      traceIds,
+      projectId,
+      timestamp: timestamp ? convertDateToClickhouseDateTime(timestamp) : null,
+    },
+  });
+
+  return records.map(convertClickhouseToDomain);
 };
 
 export const getTraceByIdOrThrow = async (
   traceId: string,
   projectId: string,
+  timestamp?: Date,
 ) => {
-  const query = `SELECT * FROM traces where id = {traceId: String} and project_id = {projectId: String} order by event_ts desc LIMIT 1 by id, project_id`;
+  const query = `SELECT * 
+      FROM traces
+      WHERE id = {traceId: String} 
+        AND project_id = {projectId: String}
+        ${timestamp ? `AND timestamp = {timestamp: DateTime64(3)}` : ""} 
+      ORDER BY event_ts DESC LIMIT 1 by id, project_id`;
+
   const records = await queryClickhouse<TraceRecordReadType>({
     query,
-    params: { traceId, projectId },
+    params: {
+      traceId,
+      projectId,
+      timestamp: timestamp ? timestamp.getTime() : null,
+    },
   });
 
   const res = records.map(convertClickhouseToDomain);
 
-  if (res.length !== 1) {
-    const errorMessage = `Trace not found or multiple traces found for traceId: ${traceId}, projectId: ${projectId}`;
+  if (res.length === 0) {
+    const errorMessage = `Trace not found for traceId: ${traceId}, projectId: ${projectId}`;
     logger.error(errorMessage);
     throw new Error(errorMessage);
   }
@@ -220,13 +347,11 @@ export const getTraceByIdOrThrow = async (
 
 export const getTracesGroupedByName = async (
   projectId: string,
+  tableDefinitions: UiColumnMapping[] = tracesTableUiColumnDefinitions,
   timestampFilter?: FilterState,
 ) => {
   const chFilter = timestampFilter
-    ? createFilterFromFilterState(
-        timestampFilter,
-        tracesTableUiColumnDefinitions,
-      )
+    ? createFilterFromFilterState(timestampFilter, tableDefinitions)
     : undefined;
 
   const timestampFilterRes = chFilter
@@ -235,18 +360,20 @@ export const getTracesGroupedByName = async (
 
   const query = `
       select 
-        name as value
+        name as name,
+        count(*) as count
       from traces t final
       WHERE t.project_id = {projectId: String}
       AND t.name IS NOT NULL
       ${timestampFilterRes?.query ? `AND ${timestampFilterRes.query}` : ""}
       GROUP BY name
-      ORDER BY name desc
+      ORDER BY count(*) desc
       LIMIT 1000;
     `;
 
   const rows = await queryClickhouse<{
-    value: string;
+    name: string;
+    count: string;
   }>({
     query: query,
     params: {
@@ -258,27 +385,69 @@ export const getTracesGroupedByName = async (
   return rows;
 };
 
-export const getTracesGroupedByTags = async (
+export const getTracesGroupedByUsers = async (
   projectId: string,
-  timestampFilter?: FilterState,
+  filter: FilterState,
+  columns?: UiColumnMapping[],
 ) => {
-  const chFilter = timestampFilter
-    ? createFilterFromFilterState(
-        timestampFilter,
-        tracesTableUiColumnDefinitions,
-      )
-    : undefined;
+  const chFilter = createFilterFromFilterState(
+    filter,
+    columns ?? tracesTableUiColumnDefinitions,
+  );
 
-  const timestampFilterRes = chFilter
-    ? new FilterList(chFilter).apply()
-    : undefined;
+  const filterRes = new FilterList(chFilter).apply();
+
+  const query = `
+      select 
+        user_id as user,
+        count(*) as count
+      from traces t final
+      WHERE t.project_id = {projectId: String}
+      AND t.user_id IS NOT NULL
+      ${filterRes?.query ? `AND ${filterRes.query}` : ""}
+      GROUP BY user
+      ORDER BY count desc
+      LIMIT 1000;
+    `;
+
+  const rows = await queryClickhouse<{
+    user: string;
+    count: string;
+  }>({
+    query: query,
+    params: {
+      projectId: projectId,
+      ...(filterRes ? filterRes.params : {}),
+    },
+  });
+
+  return rows;
+};
+
+export type GroupedTracesQueryProp = {
+  projectId: string;
+  filter: FilterState;
+  sessionIdNullFilter?: boolean;
+  columns?: UiColumnMapping[];
+};
+
+export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
+  const { projectId, filter, sessionIdNullFilter, columns } = props;
+
+  const chFilter = createFilterFromFilterState(
+    filter,
+    columns ?? tracesTableUiColumnDefinitions,
+  );
+
+  const filterRes = new FilterList(chFilter).apply();
 
   const query = `
       select 
         distinct(arrayJoin(tags)) as value
       from traces t final
       WHERE t.project_id = {projectId: String}
-      ${timestampFilterRes?.query ? `AND ${timestampFilterRes.query}` : ""}
+      ${sessionIdNullFilter ? "AND t.session_id IS NOT NULL" : ""}
+      ${filterRes?.query ? `AND ${filterRes.query}` : ""}
       LIMIT 1000;
     `;
 
@@ -288,9 +457,282 @@ export const getTracesGroupedByTags = async (
     query: query,
     params: {
       projectId: projectId,
-      ...(timestampFilterRes ? timestampFilterRes.params : {}),
+      ...(filterRes ? filterRes.params : {}),
     },
   });
 
   return rows;
 };
+
+export const getTracesGroupedByUserIds = async (
+  props: GroupedTracesQueryProp,
+) => {
+  const {
+    projectId,
+    filter,
+    sessionIdNullFilter: sessionIdNotNullFilter,
+    columns,
+  } = props;
+
+  const chFilter = createFilterFromFilterState(
+    filter,
+    columns ?? tracesTableUiColumnDefinitions,
+  );
+
+  const appliedFilter = new FilterList(chFilter).apply();
+
+  const query = `
+      select distinct user_id as user_id
+      from traces t final
+      WHERE t.project_id = {projectId: String}
+      ${sessionIdNotNullFilter ? "AND t.session_id IS NOT NULL" : ""}
+      ${appliedFilter?.query ? `AND ${appliedFilter.query}` : ""}
+      LIMIT 1000;
+    `;
+
+  const rows = await queryClickhouse<{
+    user_id: string;
+  }>({
+    query: query,
+    params: {
+      projectId: projectId,
+      ...(appliedFilter ? appliedFilter.params : {}),
+    },
+  });
+
+  return rows;
+};
+
+export type SessionDataReturnType = {
+  session_id: string;
+  max_timestamp: string;
+  min_timestamp: string;
+  trace_ids: string[];
+  user_ids: string[];
+  trace_count: number;
+  trace_tags: string[];
+  total_observations: number;
+  duration: number;
+  session_usage_details: Record<string, number>;
+  session_cost_details: Record<string, number>;
+  session_input_cost: string;
+  session_output_cost: string;
+  session_total_cost: string;
+  session_input_usage: string;
+  session_output_usage: string;
+  session_total_usage: string;
+};
+
+export const getSessionsTableCount = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  offset?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<{ count: string }>({
+    select: `
+      count(session_id) as count
+    `,
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    offset: props.offset,
+  });
+
+  return rows.length > 0 ? Number(rows[0].count) : 0;
+};
+
+export const getSessionsTable = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  offset?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<SessionDataReturnType>({
+    select: `
+    session_id, 
+    max_timestamp, 
+    min_timestamp, 
+    trace_ids, 
+    user_ids, 
+    trace_count, 
+    trace_tags,
+    total_observations,
+    duration,
+    session_usage_details,
+    session_cost_details,
+    session_input_cost,
+    session_output_cost,
+    session_total_cost,
+    session_input_usage,
+    session_output_usage,
+    session_total_usage
+    `,
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    offset: props.offset,
+  });
+
+  return rows;
+};
+
+const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
+  const { select, projectId, filter, orderBy, limit, offset } = props;
+
+  const { tracesFilter, scoresFilter, observationsFilter } =
+    getProjectIdDefaultFilter(projectId, { tracesPrefix: "s" });
+
+  tracesFilter.push(...createFilterFromFilterState(filter, sessionCols));
+
+  const tracesFilterRes = tracesFilter.apply();
+  const scoresAvgFilterRes = scoresFilter.apply();
+  const observationsStatsRes = observationsFilter.apply();
+
+  const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
+    (f) =>
+      f.field === "min_timestamp" &&
+      (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  const singleTraceFilter = traceTimestampFilter
+    ? new FilterList([
+        new DateTimeFilter({
+          clickhouseTable: "traces",
+          field: "timestamp",
+          operator: traceTimestampFilter.operator,
+          value: traceTimestampFilter.value,
+        }),
+      ]).apply()
+    : undefined;
+
+  const query = `
+      WITH observations_agg AS (
+        SELECT o.trace_id,
+              count(*) as obs_count,
+              min(o.start_time) as min_start_time,
+              max(o.end_time) as max_end_time,
+              sumMap(usage_details) as sum_usage_details,
+              sumMap(cost_details) as sum_cost_details,
+              anyLast(project_id) as project_id
+        FROM observations o FINAL
+        WHERE o.project_id = {projectId: String}
+        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+        GROUP BY o.trace_id
+    ),
+    session_data AS (
+        SELECT
+            t.session_id,
+            anyLast(t.project_id) as project_id,
+            max(t.timestamp) as max_timestamp,
+            min(t.timestamp) as min_timestamp,
+            groupArray(t.id) AS trace_ids,
+            groupUniqArray(t.user_id) AS user_ids,
+            count(*) as trace_count,
+            groupUniqArrayArray(t.tags) as trace_tags,
+            -- Aggregate observations data at session level
+            sum(o.obs_count) as total_observations,
+            date_diff('milliseconds', min(min_start_time), max(max_end_time)) as duration,
+            sumMap(o.sum_usage_details) as session_usage_details,
+            sumMap(o.sum_cost_details) as session_cost_details,
+            sumMap(o.sum_cost_details)['input'] as session_input_cost,
+            sumMap(o.sum_cost_details)['output'] as session_output_cost,
+            sumMap(o.sum_cost_details)['total'] as session_total_cost,
+            sumMap(o.sum_usage_details)['input'] as session_input_usage,
+            sumMap(o.sum_usage_details)['output'] as session_output_usage,
+            sumMap(o.sum_usage_details)['total'] as session_total_usage
+        FROM traces t FINAL
+        LEFT JOIN observations_agg o ON t.id = o.trace_id AND t.project_id = o.project_id
+        WHERE t.session_id IS NOT NULL
+            AND t.project_id = {projectId: String}
+            ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+        GROUP BY t.session_id
+    )
+    SELECT ${select}
+    FROM session_data s
+    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
+    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
+    ${limit !== undefined && offset !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+    `;
+
+  const obsStartTimeValue = traceTimestampFilter
+    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
+    : null;
+
+  const res = await queryClickhouse<T>({
+    query: query,
+    params: {
+      projectId,
+      limit: limit,
+      offset: offset,
+      ...tracesFilterRes.params,
+      ...observationsStatsRes.params,
+      ...scoresAvgFilterRes.params,
+      ...singleTraceFilter?.params,
+      ...(obsStartTimeValue
+        ? { observationsStartTime: obsStartTimeValue }
+        : {}),
+    },
+  });
+
+  return res;
+};
+
+export const getTracesForSession = async (
+  projectId: string,
+  sessionId: string,
+) => {
+  const query = `
+    SELECT
+      id,
+      user_id,
+      name,
+      timestamp,
+      project_id
+      FROM traces
+      WHERE (project_id = {projectId: String}) AND (session_id = {sessionId: String})
+      ORDER BY timestamp ASC
+      LIMIT 1 BY
+          id,
+          project_id;
+  `;
+
+  const rows = await queryClickhouse<{
+    id: string;
+    user_id: string;
+    name: string;
+    timestamp: string;
+  }>({
+    query: query,
+    params: {
+      projectId,
+      sessionId,
+    },
+  });
+
+  return rows.map((row) => ({
+    id: row.id,
+    userId: row.user_id,
+    name: row.name,
+    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
+  }));
+};
+
+export const deleteTraces = async (projectId: string, traceIds: string[]) => {
+  const query = `
+    DELETE FROM traces
+    WHERE project_id = {projectId: String}
+    AND id IN ({traceIds: Array(String)});
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      traceIds,
+    },
+  });
+};

packages/shared/src/server/repositories/traces_converters.ts

@@ -0,0 +1,128 @@
+import { ObservationLevel, Trace } from "@prisma/client";
+import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
+import { TraceRecordReadType } from "./definitions";
+import { TracesTableReturnType } from "./traces";
+import Decimal from "decimal.js";
+import { ScoreAggregate } from "../../features/scores";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+
+export const convertTraceDomainToClickhouse = (
+  trace: Trace,
+): TraceRecordReadType => {
+  return {
+    id: trace.id,
+    timestamp: convertDateToClickhouseDateTime(trace.timestamp),
+    name: trace.name,
+    user_id: trace.userId,
+    metadata: trace.metadata as Record<string, string>,
+    release: trace.release,
+    version: trace.version,
+    project_id: trace.projectId,
+    public: trace.public,
+    bookmarked: trace.bookmarked,
+    tags: trace.tags,
+    input: trace.input as string,
+    output: trace.output as string,
+    session_id: trace.sessionId,
+    created_at: convertDateToClickhouseDateTime(trace.createdAt),
+    updated_at: convertDateToClickhouseDateTime(trace.updatedAt),
+    event_ts: convertDateToClickhouseDateTime(new Date()),
+    is_deleted: 0,
+  };
+};
+
+export const convertClickhouseToDomain = (
+  record: TraceRecordReadType,
+): Trace => {
+  return {
+    id: record.id,
+    projectId: record.project_id,
+    name: record.name ?? null,
+    timestamp: parseClickhouseUTCDateTimeFormat(record.timestamp),
+    tags: record.tags,
+    bookmarked: record.bookmarked,
+    release: record.release ?? null,
+    version: record.version ?? null,
+    userId: record.user_id ?? null,
+    sessionId: record.session_id ?? null,
+    public: record.public,
+    input: record.input ?? null,
+    output: record.output ?? null,
+    metadata: record.metadata,
+    createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
+    updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
+    externalId: null,
+  };
+};
+
+export type TracesAllReturnType = {
+  id: string;
+  timestamp: Date;
+  name: string | null;
+  projectId: string;
+  userId: string | null;
+  release: string | null;
+  version: string | null;
+  public: boolean;
+  bookmarked: boolean;
+  sessionId: string | null;
+  tags: string[];
+};
+
+export const convertToReturnType = (
+  row: TracesTableReturnType,
+): TracesAllReturnType => {
+  return {
+    id: row.id,
+    name: row.name ?? null,
+    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
+    tags: row.tags,
+    bookmarked: row.bookmarked,
+    release: row.release ?? null,
+    version: row.version ?? null,
+    projectId: row.project_id,
+    userId: row.user_id ?? null,
+    sessionId: row.session_id ?? null,
+    public: row.public,
+  };
+};
+
+export type TracesMetricsReturnType = {
+  id: string;
+  promptTokens: bigint;
+  completionTokens: bigint;
+  totalTokens: bigint;
+  latency: number | null;
+  level: ObservationLevel;
+  observationCount: bigint;
+  calculatedTotalCost: Decimal | null;
+  calculatedInputCost: Decimal | null;
+  calculatedOutputCost: Decimal | null;
+  scores: ScoreAggregate;
+};
+
+export const convertMetricsReturnType = (
+  row: TracesTableReturnType & { scores: ScoreAggregate },
+): TracesMetricsReturnType => {
+  return {
+    id: row.id,
+    promptTokens: BigInt(row.usage_details?.input ?? 0),
+    completionTokens: BigInt(row.usage_details?.output ?? 0),
+    totalTokens: BigInt(row.usage_details?.total ?? 0),
+    latency: row.latency_milliseconds
+      ? Number(row.latency_milliseconds) / 1000
+      : null,
+    level: row.level,
+    observationCount: BigInt(row.observation_count ?? 0),
+    calculatedTotalCost: row.cost_details?.total
+      ? new Decimal(row.cost_details.total)
+      : null,
+    calculatedInputCost: row.cost_details?.input
+      ? new Decimal(row.cost_details.input)
+      : null,
+    calculatedOutputCost: row.cost_details?.output
+      ? new Decimal(row.cost_details.output)
+      : null,
+    scores: row.scores,
+  };
+};

packages/shared/src/server/services/PromptService.ts

@@ -30,7 +30,7 @@ export class PromptService {
       );
 
       if (cachedPrompt) {
-        this.logInfo("Returning cached prompt for params", params);
+        this.logDebug("Returning cached prompt for params", params);
 
         return cachedPrompt;
       }
@@ -223,6 +223,10 @@ export class PromptService {
     logger.info(`[PromptService] ${message}`, ...args);
   }
 
+  private logDebug(message: string, ...args: any[]) {
+    logger.debug(`[PromptService] ${message}`, ...args);
+  }
+
   private incrementMetric(name: Metrics, value: number = 1) {
     try {
       this.metricIncrementer?.(name, value);

packages/shared/src/tableDefinitions/index.ts

@@ -2,3 +2,5 @@ export * from "./sessionsView";
 export * from "./types";
 export * from "./mapObservationsTable";
 export * from "./mapTracesTable";
+export * from "./mapDashboards";
+export * from "./mapScoresTable";

packages/shared/src/tableDefinitions/mapDashboards.ts

@@ -0,0 +1,76 @@
+import { UiColumnMapping } from "./types";
+
+export const dashboardColumnDefinitions: UiColumnMapping[] = [
+  {
+    uiTableName: "Trace Name",
+    uiTableId: "traceName",
+    clickhouseTableName: "traces",
+    clickhouseSelect: 't."name"',
+  },
+  {
+    uiTableName: "Tags",
+    uiTableId: "traceTags",
+    clickhouseTableName: "traces",
+    clickhouseSelect: 't."tags"',
+  },
+  {
+    uiTableName: "Timestamp",
+    uiTableId: "timestamp",
+    clickhouseTableName: "traces",
+    clickhouseSelect: 't."timestamp"',
+  },
+  {
+    clickhouseTableName: "scores",
+    clickhouseSelect: "timestamp",
+    uiTableId: "scoreTimestamp",
+    uiTableName: "Score Timestamp",
+  },
+  {
+    clickhouseTableName: "scores",
+    clickhouseSelect: "data_type",
+    uiTableId: "scoreDataType",
+    uiTableName: "Scores Data Type",
+  },
+  {
+    clickhouseTableName: "scores",
+    clickhouseSelect: "value",
+    uiTableId: "value",
+    uiTableName: "value",
+  },
+  {
+    clickhouseTableName: "observations",
+    clickhouseSelect: "o.start_time",
+    uiTableId: "startTime",
+    uiTableName: "Start Time",
+  },
+  {
+    clickhouseTableName: "observations",
+    clickhouseSelect: "o.end_time",
+    uiTableId: "endTime",
+    uiTableName: "End Time",
+  },
+  {
+    clickhouseTableName: "observations",
+    clickhouseSelect: "o.type",
+    uiTableId: "type",
+    uiTableName: "Type",
+  },
+  {
+    clickhouseTableName: "traces",
+    clickhouseSelect: "t.user_id",
+    uiTableId: "userId",
+    uiTableName: "User",
+  },
+  {
+    clickhouseTableName: "traces",
+    clickhouseSelect: "t.release",
+    uiTableId: "release",
+    uiTableName: "Release",
+  },
+  {
+    clickhouseTableName: "traces",
+    clickhouseSelect: "t.version",
+    uiTableId: "version",
+    uiTableName: "Version",
+  },
+];

packages/shared/src/tableDefinitions/mapObservationsTable.ts

@@ -3,13 +3,41 @@
 
 import { UiColumnMapping } from "./types";
 
+export const observationsTableTraceUiColumnDefinitions: UiColumnMapping[] = [
+  {
+    uiTableName: "Trace Tags",
+    uiTableId: "traceTags",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "t.tags",
+  },
+  {
+    uiTableName: "User ID",
+    uiTableId: "userId",
+    clickhouseTableName: "traces",
+    clickhouseSelect: 't."user_id"',
+  },
+  {
+    uiTableName: "Trace Name",
+    uiTableId: "traceName",
+    clickhouseTableName: "observations",
+    clickhouseSelect: 't."name"',
+  },
+];
+
 export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
+  ...observationsTableTraceUiColumnDefinitions,
   {
     uiTableName: "ID",
     uiTableId: "id",
     clickhouseTableName: "observations",
     clickhouseSelect: 'o."id"',
   },
+  {
+    uiTableName: "Type",
+    uiTableId: "type",
+    clickhouseTableName: "observations",
+    clickhouseSelect: 'o."type"',
+  },
   {
     uiTableName: "Name",
     uiTableId: "name",
@@ -22,18 +50,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "observations",
     clickhouseSelect: 'o."trace_id"',
   },
-  {
-    uiTableName: "Trace Name",
-    uiTableId: "traceName",
-    clickhouseTableName: "observations",
-    clickhouseSelect: 't."name"',
-  },
-  {
-    uiTableName: "User ID",
-    uiTableId: "userId",
-    clickhouseTableName: "traces",
-    clickhouseSelect: 't."user_id"',
-  },
+
   {
     uiTableName: "Start Time",
     uiTableId: "startTime",
@@ -51,7 +68,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "timeToFirstToken",
     clickhouseTableName: "observations",
     clickhouseSelect:
-      "if(isNull(completion_start_time), NULL,  date_diff('seconds', start_time, completion_start_time))}",
+      "if(isNull(completion_start_time), NULL,  date_diff('seconds', start_time, completion_start_time))",
   },
   {
     uiTableName: "Latency (s)",
@@ -65,7 +82,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "tokensPerSecond",
     clickhouseTableName: "observations",
     clickhouseSelect:
-      " if(isNull(end_time) && mapExists((k, v) -> (k = 'input'), usage_details) != 1, NULL, usage_details['input'] / date_diff('seconds', start_time, end_time)",
+      "usage_details['input'] / date_diff('seconds', start_time, end_time)",
   },
   {
     uiTableName: "Input Cost ($)",
@@ -104,7 +121,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableName: "Model",
     uiTableId: "model",
     clickhouseTableName: "observations",
-    clickhouseSelect: 'o."model"',
+    clickhouseSelect: 'o."provided_model_name"',
   },
   {
     uiTableName: "Input Tokens",
@@ -155,19 +172,13 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
   {
     uiTableName: "Prompt Name",
     uiTableId: "promptName",
-    clickhouseTableName: "prompts",
-    clickhouseSelect: "p.name",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "o.prompt_name",
   },
   {
     uiTableName: "Prompt Version",
     uiTableId: "promptVersion",
-    clickhouseTableName: "prompts",
-    clickhouseSelect: "p.version",
-  },
-  {
-    uiTableName: "Trace Tags",
-    uiTableId: "traceTags",
-    clickhouseTableName: "traces",
-    clickhouseSelect: "t.tags",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "o.prompt_version",
   },
 ];

packages/shared/src/tableDefinitions/mapScoresTable.ts

@@ -0,0 +1,88 @@
+import { UiColumnMapping } from "./types";
+
+export const scoresTableUiColumnDefinitions: UiColumnMapping[] = [
+  {
+    uiTableName: "ID",
+    uiTableId: "id",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "id",
+  },
+  {
+    uiTableName: "Timestamp",
+    uiTableId: "timestamp",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "timestamp",
+  },
+  {
+    uiTableName: "Trace ID",
+    uiTableId: "traceId",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "trace_id",
+  },
+  {
+    uiTableName: "Observation ID",
+    uiTableId: "observationId",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "observation_id",
+  },
+  {
+    uiTableName: "Name",
+    uiTableId: "name",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "name",
+  },
+  {
+    uiTableName: "Value",
+    uiTableId: "value",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "value",
+  },
+  {
+    uiTableName: "Source",
+    uiTableId: "source",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "source",
+  },
+  {
+    uiTableName: "Comment",
+    uiTableId: "comment",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "comment",
+  },
+  {
+    uiTableName: "Author User ID",
+    uiTableId: "authorUserId",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "author_user_id",
+  },
+  {
+    uiTableName: "Data Type",
+    uiTableId: "dataType",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "data_type",
+  },
+  {
+    uiTableName: "String Value",
+    uiTableId: "stringValue",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "string_value",
+  },
+  {
+    uiTableName: "Trace Name",
+    uiTableId: "trace_name",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "t.name",
+  },
+  {
+    uiTableName: "User ID",
+    uiTableId: "user_id",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "t.user_id",
+  },
+  {
+    uiTableName: "Trace Tags",
+    uiTableId: "trace_tags",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "t.tags",
+  },
+];

packages/shared/src/tableDefinitions/mapSessionTable.ts

@@ -0,0 +1,126 @@
+import { UiColumnMapping } from ".";
+
+export const sessionCols: UiColumnMapping[] = [
+  // we do not access the traces scores in clichouse. We default back to the trace timestamps.
+
+  {
+    uiTableName: "⭐️",
+    uiTableId: "bookmarked",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "bookmarked",
+  },
+  {
+    uiTableName: "Created At",
+    uiTableId: "createdAt",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "min_timestamp",
+  },
+  {
+    uiTableName: "User IDs",
+    uiTableId: "userIds",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "user_ids",
+  },
+  {
+    uiTableName: "Session Duration",
+    uiTableId: "sessionDuration",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "duration",
+  },
+  {
+    uiTableName: "Count Traces",
+    uiTableId: "countTraces",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "trace_count",
+  },
+  {
+    uiTableName: "Session Input Cost",
+    uiTableId: "inputCost",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_input_cost",
+  },
+  {
+    uiTableName: "Session Output Cost",
+    uiTableId: "outputCost",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_output_cost",
+  },
+  {
+    uiTableName: "Session Total Cost",
+    uiTableId: "totalCost",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_total_cost",
+  },
+  {
+    uiTableName: "Input Tokens",
+    uiTableId: "inputTokens",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_input_usage",
+  },
+  {
+    uiTableName: "Output Tokens",
+    uiTableId: "outputTokens",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_output_usage",
+  },
+  {
+    uiTableName: "Total Tokens",
+    uiTableId: "totalTokens",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_total_usage",
+  },
+  {
+    uiTableName: "Usage",
+    uiTableId: "totalTokens",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_total_usage",
+  },
+  {
+    uiTableName: "Session Total Usage",
+    uiTableId: "usage",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_total_usage",
+  },
+  {
+    uiTableName: "Session Duration (s)",
+    uiTableId: "sessionDuration",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "duration",
+  },
+  {
+    uiTableName: "Traces Count",
+    uiTableId: "tracesCount",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "trace_count",
+  },
+  {
+    uiTableName: "Input Cost ($)",
+    uiTableId: "inputCost",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_input_cost",
+  },
+  {
+    uiTableName: "Output Cost ($)",
+    uiTableId: "outputCost",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_output_cost",
+  },
+  {
+    uiTableName: "Total Cost ($)",
+    uiTableId: "totalCost",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_total_cost",
+  },
+  {
+    uiTableName: "Trace Tags",
+    uiTableId: "traceTags",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "trace_tags",
+  },
+  {
+    uiTableName: "ID",
+    uiTableId: "sessionId",
+    clickhouseTableName: "traces",
+    clickhouseSelect: "session_id",
+  },
+];

packages/shared/src/tableDefinitions/mapTracesTable.ts

@@ -2,7 +2,7 @@ import { UiColumnMapping } from "./types";
 
 export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
   {
-    uiTableName: "bookmarked",
+    uiTableName: "⭐️",
     uiTableId: "bookmarked",
     clickhouseTableName: "traces",
     clickhouseSelect: "bookmarked",
@@ -89,6 +89,19 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseSelect:
       "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
   },
+  {
+    uiTableName: "Usage",
+    uiTableId: "totalTokens",
+    clickhouseTableName: "traces",
+    clickhouseSelect:
+      "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
+  },
+  {
+    uiTableName: "Scores",
+    uiTableId: "scores",
+    clickhouseTableName: "scores",
+    clickhouseSelect: "s.scores_avg",
+  },
   {
     uiTableName: "Latency (s)",
     uiTableId: "latency",

pnpm-lock.yaml

@@ -101,7 +101,7 @@ importers:
         version: 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       '@vercel/style-guide':
         specifier: ^6.0.0
-        version: 6.0.0(@next/eslint-plugin-next@14.2.15)(eslint@8.57.0)(jest@29.7.0)(prettier@3.3.3)(typescript@5.4.5)(vitest@2.1.2)
+        version: 6.0.0(@next/eslint-plugin-next@14.2.15)(eslint@8.57.0)(jest@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0))(prettier@3.3.3)(typescript@5.4.5)(vitest@2.1.2(@types/node@20.14.8)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1))
       eslint-config-next:
         specifier: ^14.2.15
         version: 14.2.15(eslint@8.57.0)(typescript@5.4.5)
@@ -572,7 +572,7 @@ importers:
         version: 0.27.4
       langchain:
         specifier: ^0.3.2
-        version: 0.3.2(@langchain/anthropic@0.3.1(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8))))(@langchain/aws@0.1.0(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8))))(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))(axios@1.7.7)(handlebars@4.7.8)(openai@4.62.1(zod@3.23.8))
+        version: 0.3.2(@langchain/anthropic@0.3.1(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8))))(@langchain/aws@0.1.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8))))(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))(axios@1.7.7)(handlebars@4.7.8)(openai@4.62.1(zod@3.23.8))
       lodash:
         specifier: ^4.17.21
         version: 4.17.21
@@ -688,7 +688,7 @@ importers:
         version: 0.5.7(tailwindcss@3.4.13(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))
       '@testing-library/jest-dom':
         specifier: ^6.4.6
-        version: 6.4.6(@jest/globals@29.7.0)(@types/jest@29.5.12)(jest@29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))(vitest@2.1.2(@types/node@20.10.5))
+        version: 6.4.6(@jest/globals@29.7.0)(@types/jest@29.5.12)(jest@29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))(vitest@2.1.2(@types/node@20.10.5)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1))
       '@testing-library/react':
         specifier: ^15.0.7
         version: 15.0.7(@types/react@18.2.79)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
@@ -945,9 +945,6 @@ importers:
       prettier:
         specifier: ^3.3.3
         version: 3.3.3
-      rus-diff:
-        specifier: ^1.1.0
-        version: 1.1.0
       ts-node:
         specifier: ^10.9.2
         version: 10.9.2(@types/node@20.11.29)(typescript@5.4.5)
@@ -8100,9 +8097,6 @@ packages:
   json-buffer@3.0.1:
     resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==}
 
-  json-hash@1.2.0:
-    resolution: {integrity: sha512-lL0pLS/jguRLPZZZUEoNNoOxXtBHiM+iNsaTbPG0W8s/HpGuRNOw+LztiYaSo6Q8YWShEyIM++jM1jvvNcl0tw==}
-
   json-parse-even-better-errors@2.3.1:
     resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
 
@@ -10041,9 +10035,6 @@ packages:
   run-parallel@1.2.0:
     resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}
 
-  rus-diff@1.1.0:
-    resolution: {integrity: sha512-h2SBIrpYQ0eevfvghEA67QQbc3fD2KJl/aERQuvi8AF8g0FSvZpSEwIeJgcoBDteFB5TmXtGkw6aLn5uhMYxYw==}
-
   rusha@0.8.14:
     resolution: {integrity: sha512-cLgakCUf6PedEu15t8kbsjnwIFFR2D4RfL+W3iWFJ4iac7z4B0ZI8fxy4R3J956kAI68HclCFGL8MPoUVC3qVA==}
 
@@ -11620,7 +11611,7 @@ snapshots:
       '@aws-sdk/client-sso-oidc': 3.675.0(@aws-sdk/client-sts@3.675.0)
       '@aws-sdk/client-sts': 3.675.0
       '@aws-sdk/core': 3.667.0
-      '@aws-sdk/credential-provider-node': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)
+      '@aws-sdk/credential-provider-node': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))(@aws-sdk/client-sts@3.675.0)
       '@aws-sdk/middleware-host-header': 3.667.0
       '@aws-sdk/middleware-logger': 3.667.0
       '@aws-sdk/middleware-recursion-detection': 3.667.0
@@ -11716,7 +11707,7 @@ snapshots:
       '@aws-sdk/client-sso-oidc': 3.675.0(@aws-sdk/client-sts@3.675.0)
       '@aws-sdk/client-sts': 3.675.0
       '@aws-sdk/core': 3.667.0
-      '@aws-sdk/credential-provider-node': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)
+      '@aws-sdk/credential-provider-node': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))(@aws-sdk/client-sts@3.675.0)
       '@aws-sdk/middleware-bucket-endpoint': 3.667.0
       '@aws-sdk/middleware-expect-continue': 3.667.0
       '@aws-sdk/middleware-flexible-checksums': 3.669.0
@@ -12112,7 +12103,7 @@ snapshots:
       '@aws-sdk/credential-provider-env': 3.667.0
       '@aws-sdk/credential-provider-http': 3.667.0
       '@aws-sdk/credential-provider-process': 3.667.0
-      '@aws-sdk/credential-provider-sso': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))
+      '@aws-sdk/credential-provider-sso': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))
       '@aws-sdk/credential-provider-web-identity': 3.667.0(@aws-sdk/client-sts@3.668.0)
       '@aws-sdk/types': 3.667.0
       '@smithy/credential-provider-imds': 3.2.4
@@ -12124,14 +12115,14 @@ snapshots:
       - '@aws-sdk/client-sso-oidc'
       - aws-crt
 
-  '@aws-sdk/credential-provider-ini@3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)':
+  '@aws-sdk/credential-provider-ini@3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)':
     dependencies:
       '@aws-sdk/client-sts': 3.675.0
       '@aws-sdk/core': 3.667.0
       '@aws-sdk/credential-provider-env': 3.667.0
       '@aws-sdk/credential-provider-http': 3.667.0
       '@aws-sdk/credential-provider-process': 3.667.0
-      '@aws-sdk/credential-provider-sso': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))
+      '@aws-sdk/credential-provider-sso': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))
       '@aws-sdk/credential-provider-web-identity': 3.667.0(@aws-sdk/client-sts@3.675.0)
       '@aws-sdk/types': 3.667.0
       '@smithy/credential-provider-imds': 3.2.4
@@ -12142,15 +12133,17 @@ snapshots:
     transitivePeerDependencies:
       - '@aws-sdk/client-sso-oidc'
       - aws-crt
+    optional: true
 
-  '@aws-sdk/credential-provider-node@3.668.0':
+  '@aws-sdk/credential-provider-ini@3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)':
     dependencies:
+      '@aws-sdk/client-sts': 3.675.0
+      '@aws-sdk/core': 3.667.0
       '@aws-sdk/credential-provider-env': 3.667.0
       '@aws-sdk/credential-provider-http': 3.667.0
-      '@aws-sdk/credential-provider-ini': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))(@aws-sdk/client-sts@3.668.0)
       '@aws-sdk/credential-provider-process': 3.667.0
-      '@aws-sdk/credential-provider-sso': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))
-      '@aws-sdk/credential-provider-web-identity': 3.667.0(@aws-sdk/client-sts@3.668.0)
+      '@aws-sdk/credential-provider-sso': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))
+      '@aws-sdk/credential-provider-web-identity': 3.667.0(@aws-sdk/client-sts@3.675.0)
       '@aws-sdk/types': 3.667.0
       '@smithy/credential-provider-imds': 3.2.4
       '@smithy/property-provider': 3.1.7
@@ -12159,9 +12152,7 @@ snapshots:
       tslib: 2.6.3
     transitivePeerDependencies:
       - '@aws-sdk/client-sso-oidc'
-      - '@aws-sdk/client-sts'
       - aws-crt
-    optional: true
 
   '@aws-sdk/credential-provider-node@3.668.0(@aws-sdk/client-sso-oidc@3.668.0(@aws-sdk/client-sts@3.668.0))(@aws-sdk/client-sts@3.668.0)':
     dependencies:
@@ -12188,7 +12179,7 @@ snapshots:
       '@aws-sdk/credential-provider-http': 3.667.0
       '@aws-sdk/credential-provider-ini': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))(@aws-sdk/client-sts@3.668.0)
       '@aws-sdk/credential-provider-process': 3.667.0
-      '@aws-sdk/credential-provider-sso': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))
+      '@aws-sdk/credential-provider-sso': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))
       '@aws-sdk/credential-provider-web-identity': 3.667.0(@aws-sdk/client-sts@3.668.0)
       '@aws-sdk/types': 3.667.0
       '@smithy/credential-provider-imds': 3.2.4
@@ -12201,13 +12192,52 @@ snapshots:
       - '@aws-sdk/client-sts'
       - aws-crt
 
+  '@aws-sdk/credential-provider-node@3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)':
+    dependencies:
+      '@aws-sdk/credential-provider-env': 3.667.0
+      '@aws-sdk/credential-provider-http': 3.667.0
+      '@aws-sdk/credential-provider-ini': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)
+      '@aws-sdk/credential-provider-process': 3.667.0
+      '@aws-sdk/credential-provider-sso': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))
+      '@aws-sdk/credential-provider-web-identity': 3.667.0(@aws-sdk/client-sts@3.675.0)
+      '@aws-sdk/types': 3.667.0
+      '@smithy/credential-provider-imds': 3.2.4
+      '@smithy/property-provider': 3.1.7
+      '@smithy/shared-ini-file-loader': 3.1.8
+      '@smithy/types': 3.5.0
+      tslib: 2.6.3
+    transitivePeerDependencies:
+      - '@aws-sdk/client-sso-oidc'
+      - '@aws-sdk/client-sts'
+      - aws-crt
+    optional: true
+
+  '@aws-sdk/credential-provider-node@3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))(@aws-sdk/client-sts@3.675.0)':
+    dependencies:
+      '@aws-sdk/credential-provider-env': 3.667.0
+      '@aws-sdk/credential-provider-http': 3.667.0
+      '@aws-sdk/credential-provider-ini': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)
+      '@aws-sdk/credential-provider-process': 3.667.0
+      '@aws-sdk/credential-provider-sso': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))
+      '@aws-sdk/credential-provider-web-identity': 3.667.0(@aws-sdk/client-sts@3.675.0)
+      '@aws-sdk/types': 3.667.0
+      '@smithy/credential-provider-imds': 3.2.4
+      '@smithy/property-provider': 3.1.7
+      '@smithy/shared-ini-file-loader': 3.1.8
+      '@smithy/types': 3.5.0
+      tslib: 2.6.3
+    transitivePeerDependencies:
+      - '@aws-sdk/client-sso-oidc'
+      - '@aws-sdk/client-sts'
+      - aws-crt
+
   '@aws-sdk/credential-provider-node@3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)':
     dependencies:
       '@aws-sdk/credential-provider-env': 3.667.0
       '@aws-sdk/credential-provider-http': 3.667.0
       '@aws-sdk/credential-provider-ini': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)
       '@aws-sdk/credential-provider-process': 3.667.0
-      '@aws-sdk/credential-provider-sso': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))
+      '@aws-sdk/credential-provider-sso': 3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))
       '@aws-sdk/credential-provider-web-identity': 3.667.0(@aws-sdk/client-sts@3.675.0)
       '@aws-sdk/types': 3.667.0
       '@smithy/credential-provider-imds': 3.2.4
@@ -12243,11 +12273,11 @@ snapshots:
       - '@aws-sdk/client-sso-oidc'
       - aws-crt
 
-  '@aws-sdk/credential-provider-sso@3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))':
+  '@aws-sdk/credential-provider-sso@3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))':
     dependencies:
       '@aws-sdk/client-sso': 3.668.0
       '@aws-sdk/core': 3.667.0
-      '@aws-sdk/token-providers': 3.667.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))
+      '@aws-sdk/token-providers': 3.667.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))
       '@aws-sdk/types': 3.667.0
       '@smithy/property-provider': 3.1.7
       '@smithy/shared-ini-file-loader': 3.1.8
@@ -12257,11 +12287,11 @@ snapshots:
       - '@aws-sdk/client-sso-oidc'
       - aws-crt
 
-  '@aws-sdk/credential-provider-sso@3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))':
+  '@aws-sdk/credential-provider-sso@3.675.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))':
     dependencies:
       '@aws-sdk/client-sso': 3.675.0
       '@aws-sdk/core': 3.667.0
-      '@aws-sdk/token-providers': 3.667.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))
+      '@aws-sdk/token-providers': 3.667.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))
       '@aws-sdk/types': 3.667.0
       '@smithy/property-provider': 3.1.7
       '@smithy/shared-ini-file-loader': 3.1.8
@@ -12464,7 +12494,7 @@ snapshots:
       '@smithy/types': 3.5.0
       tslib: 2.6.3
 
-  '@aws-sdk/token-providers@3.667.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.668.0))':
+  '@aws-sdk/token-providers@3.667.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))':
     dependencies:
       '@aws-sdk/client-sso-oidc': 3.675.0(@aws-sdk/client-sts@3.675.0)
       '@aws-sdk/types': 3.667.0
@@ -13481,42 +13511,6 @@ snapshots:
       jest-util: 29.7.0
       slash: 3.0.0
 
-  '@jest/core@29.7.0':
-    dependencies:
-      '@jest/console': 29.7.0
-      '@jest/reporters': 29.7.0
-      '@jest/test-result': 29.7.0
-      '@jest/transform': 29.7.0
-      '@jest/types': 29.6.3
-      '@types/node': 20.14.8
-      ansi-escapes: 4.3.2
-      chalk: 4.1.2
-      ci-info: 3.9.0
-      exit: 0.1.2
-      graceful-fs: 4.2.11
-      jest-changed-files: 29.7.0
-      jest-config: 29.7.0(@types/node@20.14.8)
-      jest-haste-map: 29.7.0
-      jest-message-util: 29.7.0
-      jest-regex-util: 29.6.3
-      jest-resolve: 29.7.0
-      jest-resolve-dependencies: 29.7.0
-      jest-runner: 29.7.0
-      jest-runtime: 29.7.0
-      jest-snapshot: 29.7.0
-      jest-util: 29.7.0
-      jest-validate: 29.7.0
-      jest-watcher: 29.7.0
-      micromatch: 4.0.8
-      pretty-format: 29.7.0
-      slash: 3.0.0
-      strip-ansi: 6.0.1
-    transitivePeerDependencies:
-      - babel-plugin-macros
-      - supports-color
-      - ts-node
-    optional: true
-
   '@jest/core@29.7.0(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))':
     dependencies:
       '@jest/console': 29.7.0
@@ -13731,12 +13725,12 @@ snapshots:
       - '@aws-sdk/client-sts'
       - aws-crt
 
-  '@langchain/aws@0.1.0(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))':
+  '@langchain/aws@0.1.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))':
     dependencies:
       '@aws-sdk/client-bedrock-agent-runtime': 3.668.0
       '@aws-sdk/client-bedrock-runtime': 3.668.0
       '@aws-sdk/client-kendra': 3.668.0
-      '@aws-sdk/credential-provider-node': 3.668.0
+      '@aws-sdk/credential-provider-node': 3.668.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)
       '@langchain/core': 0.3.9(openai@4.62.1(zod@3.23.8))
       zod: 3.23.8
       zod-to-json-schema: 3.23.5(zod@3.23.8)
@@ -16388,7 +16382,7 @@ snapshots:
       lz-string: 1.5.0
       pretty-format: 27.5.1
 
-  '@testing-library/jest-dom@6.4.6(@jest/globals@29.7.0)(@types/jest@29.5.12)(jest@29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))(vitest@2.1.2(@types/node@20.10.5))':
+  '@testing-library/jest-dom@6.4.6(@jest/globals@29.7.0)(@types/jest@29.5.12)(jest@29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)))(vitest@2.1.2(@types/node@20.10.5)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1))':
     dependencies:
       '@adobe/css-tools': 4.4.0
       '@babel/runtime': 7.24.7
@@ -16402,7 +16396,7 @@ snapshots:
       '@jest/globals': 29.7.0
       '@types/jest': 29.5.12
       jest: 29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
-      vitest: 2.1.2(@types/node@20.10.5)
+      vitest: 2.1.2(@types/node@20.10.5)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1)
 
   '@testing-library/react@15.0.7(@types/react@18.2.79)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)':
     dependencies:
@@ -17054,7 +17048,7 @@ snapshots:
 
   '@ungap/structured-clone@1.2.0': {}
 
-  '@vercel/style-guide@6.0.0(@next/eslint-plugin-next@14.2.15)(eslint@8.57.0)(jest@29.7.0)(prettier@3.3.3)(typescript@5.4.5)(vitest@2.1.2)':
+  '@vercel/style-guide@6.0.0(@next/eslint-plugin-next@14.2.15)(eslint@8.57.0)(jest@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0))(prettier@3.3.3)(typescript@5.4.5)(vitest@2.1.2(@types/node@20.14.8)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1))':
     dependencies:
       '@babel/core': 7.24.3
       '@babel/eslint-parser': 7.24.1(@babel/core@7.24.3)(eslint@8.57.0)
@@ -17063,18 +17057,18 @@ snapshots:
       '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       eslint-config-prettier: 9.1.0(eslint@8.57.0)
       eslint-import-resolver-alias: 1.1.2(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))
-      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1)(eslint@8.57.0)
+      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0)
       eslint-plugin-eslint-comments: 3.2.0(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
-      eslint-plugin-jest: 27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0)(typescript@5.4.5)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
+      eslint-plugin-jest: 27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0))(typescript@5.4.5)
       eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0)
-      eslint-plugin-playwright: 1.5.4(eslint-plugin-jest@27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0)(typescript@5.4.5))(eslint@8.57.0)
+      eslint-plugin-playwright: 1.5.4(eslint-plugin-jest@27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0))(typescript@5.4.5))(eslint@8.57.0)
       eslint-plugin-react: 7.34.1(eslint@8.57.0)
       eslint-plugin-react-hooks: 4.6.0(eslint@8.57.0)
       eslint-plugin-testing-library: 6.2.0(eslint@8.57.0)(typescript@5.4.5)
       eslint-plugin-tsdoc: 0.2.17
       eslint-plugin-unicorn: 51.0.1(eslint@8.57.0)
-      eslint-plugin-vitest: 0.3.26(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5)(vitest@2.1.2)
+      eslint-plugin-vitest: 0.3.26(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5)(vitest@2.1.2(@types/node@20.14.8)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1))
       prettier-plugin-packagejson: 2.4.12(prettier@3.3.3)
     optionalDependencies:
       '@next/eslint-plugin-next': 14.2.15
@@ -17095,6 +17089,16 @@ snapshots:
       chai: 5.1.1
       tinyrainbow: 1.2.0
 
+  '@vitest/mocker@2.1.2(@vitest/spy@2.1.2)(msw@2.4.9(typescript@5.4.5))(vite@5.2.14(@types/node@20.10.5)(terser@5.34.1))':
+    dependencies:
+      '@vitest/spy': 2.1.2
+      estree-walker: 3.0.3
+      magic-string: 0.30.11
+    optionalDependencies:
+      msw: 2.4.9(typescript@5.4.5)
+      vite: 5.2.14(@types/node@20.10.5)(terser@5.34.1)
+    optional: true
+
   '@vitest/mocker@2.1.2(@vitest/spy@2.1.2)(msw@2.4.9(typescript@5.4.5))(vite@5.2.14(@types/node@20.11.29)(terser@5.34.1))':
     dependencies:
       '@vitest/spy': 2.1.2
@@ -17104,13 +17108,14 @@ snapshots:
       msw: 2.4.9(typescript@5.4.5)
       vite: 5.2.14(@types/node@20.11.29)(terser@5.34.1)
 
-  '@vitest/mocker@2.1.2(@vitest/spy@2.1.2)(vite@5.2.14(@types/node@20.10.5))':
+  '@vitest/mocker@2.1.2(@vitest/spy@2.1.2)(msw@2.4.9(typescript@5.4.5))(vite@5.2.14(@types/node@20.14.8)(terser@5.34.1))':
     dependencies:
       '@vitest/spy': 2.1.2
       estree-walker: 3.0.3
       magic-string: 0.30.11
     optionalDependencies:
-      vite: 5.2.14(@types/node@20.10.5)
+      msw: 2.4.9(typescript@5.4.5)
+      vite: 5.2.14(@types/node@20.14.8)(terser@5.34.1)
     optional: true
 
   '@vitest/pretty-format@2.1.2':
@@ -18141,22 +18146,6 @@ snapshots:
       crc-32: 1.2.2
       readable-stream: 3.6.2
 
-  create-jest@29.7.0:
-    dependencies:
-      '@jest/types': 29.6.3
-      chalk: 4.1.2
-      exit: 0.1.2
-      graceful-fs: 4.2.11
-      jest-config: 29.7.0(@types/node@20.14.8)
-      jest-util: 29.7.0
-      prompts: 2.4.2
-    transitivePeerDependencies:
-      - '@types/node'
-      - babel-plugin-macros
-      - supports-color
-      - ts-node
-    optional: true
-
   create-jest@29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)):
     dependencies:
       '@jest/types': 29.6.3
@@ -18172,6 +18161,22 @@ snapshots:
       - supports-color
       - ts-node
 
+  create-jest@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0):
+    dependencies:
+      '@jest/types': 29.6.3
+      chalk: 4.1.2
+      exit: 0.1.2
+      graceful-fs: 4.2.11
+      jest-config: 29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+      jest-util: 29.7.0
+      prompts: 2.4.2
+    transitivePeerDependencies:
+      - '@types/node'
+      - babel-plugin-macros
+      - supports-color
+      - ts-node
+    optional: true
+
   create-require@1.1.1: {}
 
   crelt@1.0.6: {}
@@ -18934,7 +18939,7 @@ snapshots:
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
       eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
       eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0)
       eslint-plugin-react: 7.34.1(eslint@8.57.0)
       eslint-plugin-react-hooks: 4.6.0(eslint@8.57.0)
@@ -18951,7 +18956,7 @@ snapshots:
   eslint-config-standard@17.1.0(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.4.0(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
       eslint: 8.57.0
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
       eslint-plugin-n: 16.6.2(eslint@8.57.0)
       eslint-plugin-promise: 6.4.0(eslint@8.57.0)
 
@@ -18962,7 +18967,7 @@ snapshots:
 
   eslint-import-resolver-alias@1.1.2(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)):
     dependencies:
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
 
   eslint-import-resolver-node@0.3.9:
     dependencies:
@@ -18978,7 +18983,7 @@ snapshots:
       enhanced-resolve: 5.17.1
       eslint: 8.57.0
       eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
       fast-glob: 3.3.2
       get-tsconfig: 4.8.1
       is-core-module: 2.15.1
@@ -18989,13 +18994,13 @@ snapshots:
       - eslint-import-resolver-webpack
       - supports-color
 
-  eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1)(eslint@8.57.0):
+  eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
       debug: 4.3.7(supports-color@5.5.0)
       enhanced-resolve: 5.17.1
       eslint: 8.57.0
-      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
       fast-glob: 3.3.2
       get-tsconfig: 4.8.1
       is-core-module: 2.15.1
@@ -19017,14 +19022,14 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-module-utils@2.8.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
+  eslint-module-utils@2.8.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
       debug: 3.2.7
     optionalDependencies:
       '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1)(eslint@8.57.0)
+      eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0)
     transitivePeerDependencies:
       - supports-color
 
@@ -19041,7 +19046,7 @@ snapshots:
       eslint: 8.57.0
       ignore: 5.3.1
 
-  eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0):
+  eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
       array-includes: 3.1.7
       array.prototype.findlastindex: 1.2.4
@@ -19051,7 +19056,7 @@ snapshots:
       doctrine: 2.1.0
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
+      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
       hasown: 2.0.2
       is-core-module: 2.15.1
       is-glob: 4.0.3
@@ -19068,40 +19073,13 @@ snapshots:
       - eslint-import-resolver-webpack
       - supports-color
 
-  eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0):
-    dependencies:
-      array-includes: 3.1.7
-      array.prototype.findlastindex: 1.2.4
-      array.prototype.flat: 1.3.2
-      array.prototype.flatmap: 1.3.2
-      debug: 3.2.7
-      doctrine: 2.1.0
-      eslint: 8.57.0
-      eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
-      hasown: 2.0.2
-      is-core-module: 2.15.1
-      is-glob: 4.0.3
-      minimatch: 3.1.2
-      object.fromentries: 2.0.8
-      object.groupby: 1.0.3
-      object.values: 1.2.0
-      semver: 6.3.1
-      tsconfig-paths: 3.15.0
-    optionalDependencies:
-      '@typescript-eslint/parser': 7.12.0(eslint@8.57.0)(typescript@5.4.5)
-    transitivePeerDependencies:
-      - eslint-import-resolver-typescript
-      - eslint-import-resolver-webpack
-      - supports-color
-
-  eslint-plugin-jest@27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0)(typescript@5.4.5):
+  eslint-plugin-jest@27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0))(typescript@5.4.5):
     dependencies:
       '@typescript-eslint/utils': 5.62.0(eslint@8.57.0)(typescript@5.4.5)
       eslint: 8.57.0
     optionalDependencies:
       '@typescript-eslint/eslint-plugin': 7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5)
-      jest: 29.7.0
+      jest: 29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0)
     transitivePeerDependencies:
       - supports-color
       - typescript
@@ -19143,12 +19121,12 @@ snapshots:
 
   eslint-plugin-only-warn@1.1.0: {}
 
-  eslint-plugin-playwright@1.5.4(eslint-plugin-jest@27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0)(typescript@5.4.5))(eslint@8.57.0):
+  eslint-plugin-playwright@1.5.4(eslint-plugin-jest@27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0))(typescript@5.4.5))(eslint@8.57.0):
     dependencies:
       eslint: 8.57.0
       globals: 13.24.0
     optionalDependencies:
-      eslint-plugin-jest: 27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0)(typescript@5.4.5)
+      eslint-plugin-jest: 27.9.0(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(jest@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0))(typescript@5.4.5)
 
   eslint-plugin-prettier@5.1.3(@types/eslint@8.56.12)(eslint-config-prettier@9.1.0(eslint@8.57.0))(eslint@8.57.0)(prettier@3.3.3):
     dependencies:
@@ -19230,13 +19208,13 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-plugin-vitest@0.3.26(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5)(vitest@2.1.2):
+  eslint-plugin-vitest@0.3.26(@typescript-eslint/eslint-plugin@7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5)(vitest@2.1.2(@types/node@20.14.8)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1)):
     dependencies:
       '@typescript-eslint/utils': 7.3.1(eslint@8.57.0)(typescript@5.4.5)
       eslint: 8.57.0
     optionalDependencies:
       '@typescript-eslint/eslint-plugin': 7.3.1(@typescript-eslint/parser@7.12.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5)
-      vitest: 2.1.2(@types/node@20.11.29)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1)
+      vitest: 2.1.2(@types/node@20.14.8)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1)
     transitivePeerDependencies:
       - supports-color
       - typescript
@@ -20456,26 +20434,6 @@ snapshots:
       - babel-plugin-macros
       - supports-color
 
-  jest-cli@29.7.0:
-    dependencies:
-      '@jest/core': 29.7.0
-      '@jest/test-result': 29.7.0
-      '@jest/types': 29.6.3
-      chalk: 4.1.2
-      create-jest: 29.7.0
-      exit: 0.1.2
-      import-local: 3.1.0
-      jest-config: 29.7.0(@types/node@20.14.8)
-      jest-util: 29.7.0
-      jest-validate: 29.7.0
-      yargs: 17.7.2
-    transitivePeerDependencies:
-      - '@types/node'
-      - babel-plugin-macros
-      - supports-color
-      - ts-node
-    optional: true
-
   jest-cli@29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)):
     dependencies:
       '@jest/core': 29.7.0(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
@@ -20495,6 +20453,26 @@ snapshots:
       - supports-color
       - ts-node
 
+  jest-cli@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0):
+    dependencies:
+      '@jest/core': 29.7.0(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+      '@jest/test-result': 29.7.0
+      '@jest/types': 29.6.3
+      chalk: 4.1.2
+      create-jest: 29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0)
+      exit: 0.1.2
+      import-local: 3.1.0
+      jest-config: 29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+      jest-util: 29.7.0
+      jest-validate: 29.7.0
+      yargs: 17.7.2
+    transitivePeerDependencies:
+      - '@types/node'
+      - babel-plugin-macros
+      - supports-color
+      - ts-node
+    optional: true
+
   jest-config@29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)):
     dependencies:
       '@babel/core': 7.24.3
@@ -20526,37 +20504,6 @@ snapshots:
       - babel-plugin-macros
       - supports-color
 
-  jest-config@29.7.0(@types/node@20.14.8):
-    dependencies:
-      '@babel/core': 7.24.3
-      '@jest/test-sequencer': 29.7.0
-      '@jest/types': 29.6.3
-      babel-jest: 29.7.0(@babel/core@7.24.3)
-      chalk: 4.1.2
-      ci-info: 3.9.0
-      deepmerge: 4.3.1
-      glob: 7.2.3
-      graceful-fs: 4.2.11
-      jest-circus: 29.7.0(babel-plugin-macros@3.1.0)
-      jest-environment-node: 29.7.0
-      jest-get-type: 29.6.3
-      jest-regex-util: 29.6.3
-      jest-resolve: 29.7.0
-      jest-runner: 29.7.0
-      jest-util: 29.7.0
-      jest-validate: 29.7.0
-      micromatch: 4.0.8
-      parse-json: 5.2.0
-      pretty-format: 29.7.0
-      slash: 3.0.0
-      strip-json-comments: 3.1.1
-    optionalDependencies:
-      '@types/node': 20.14.8
-    transitivePeerDependencies:
-      - babel-plugin-macros
-      - supports-color
-    optional: true
-
   jest-config@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)):
     dependencies:
       '@babel/core': 7.24.3
@@ -20824,19 +20771,6 @@ snapshots:
       merge-stream: 2.0.0
       supports-color: 8.1.1
 
-  jest@29.7.0:
-    dependencies:
-      '@jest/core': 29.7.0
-      '@jest/types': 29.6.3
-      import-local: 3.1.0
-      jest-cli: 29.7.0
-    transitivePeerDependencies:
-      - '@types/node'
-      - babel-plugin-macros
-      - supports-color
-      - ts-node
-    optional: true
-
   jest@29.7.0(@types/node@20.10.5)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5)):
     dependencies:
       '@jest/core': 29.7.0(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
@@ -20849,6 +20783,19 @@ snapshots:
       - supports-color
       - ts-node
 
+  jest@29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0):
+    dependencies:
+      '@jest/core': 29.7.0(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.10.5)(typescript@5.4.5))
+      '@jest/types': 29.6.3
+      import-local: 3.1.0
+      jest-cli: 29.7.0(@types/node@20.14.8)(babel-plugin-macros@3.1.0)
+    transitivePeerDependencies:
+      - '@types/node'
+      - babel-plugin-macros
+      - supports-color
+      - ts-node
+    optional: true
+
   jiti@1.21.0: {}
 
   jju@1.4.0: {}
@@ -20929,8 +20876,6 @@ snapshots:
 
   json-buffer@3.0.1: {}
 
-  json-hash@1.2.0: {}
-
   json-parse-even-better-errors@2.3.1: {}
 
   json-schema-traverse@0.4.1: {}
@@ -21031,7 +20976,7 @@ snapshots:
       - encoding
       - openai
 
-  langchain@0.3.2(@langchain/anthropic@0.3.1(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8))))(@langchain/aws@0.1.0(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8))))(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))(axios@1.7.7)(handlebars@4.7.8)(openai@4.62.1(zod@3.23.8)):
+  langchain@0.3.2(@langchain/anthropic@0.3.1(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8))))(@langchain/aws@0.1.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8))))(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))(axios@1.7.7)(handlebars@4.7.8)(openai@4.62.1(zod@3.23.8)):
     dependencies:
       '@langchain/core': 0.3.9(openai@4.62.1(zod@3.23.8))
       '@langchain/openai': 0.3.0(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))
@@ -21048,7 +20993,7 @@ snapshots:
       zod-to-json-schema: 3.23.5(zod@3.23.8)
     optionalDependencies:
       '@langchain/anthropic': 0.3.1(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))
-      '@langchain/aws': 0.1.0(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))
+      '@langchain/aws': 0.1.0(@aws-sdk/client-sso-oidc@3.675.0(@aws-sdk/client-sts@3.675.0))(@aws-sdk/client-sts@3.675.0)(@langchain/core@0.3.9(openai@4.62.1(zod@3.23.8)))
       axios: 1.7.7
       handlebars: 4.7.8
     transitivePeerDependencies:
@@ -23169,10 +23114,6 @@ snapshots:
     dependencies:
       queue-microtask: 1.2.3
 
-  rus-diff@1.1.0:
-    dependencies:
-      json-hash: 1.2.0
-
   rusha@0.8.14: {}
 
   rw@1.3.3: {}
@@ -24222,12 +24163,12 @@ snapshots:
       d3-time: 3.1.0
       d3-timer: 3.0.1
 
-  vite-node@2.1.2(@types/node@20.10.5):
+  vite-node@2.1.2(@types/node@20.10.5)(terser@5.34.1):
     dependencies:
       cac: 6.7.14
       debug: 4.3.7(supports-color@5.5.0)
       pathe: 1.1.2
-      vite: 5.2.14(@types/node@20.10.5)
+      vite: 5.2.14(@types/node@20.10.5)(terser@5.34.1)
     transitivePeerDependencies:
       - '@types/node'
       - less
@@ -24255,7 +24196,24 @@ snapshots:
       - supports-color
       - terser
 
-  vite@5.2.14(@types/node@20.10.5):
+  vite-node@2.1.2(@types/node@20.14.8)(terser@5.34.1):
+    dependencies:
+      cac: 6.7.14
+      debug: 4.3.7(supports-color@5.5.0)
+      pathe: 1.1.2
+      vite: 5.2.14(@types/node@20.14.8)(terser@5.34.1)
+    transitivePeerDependencies:
+      - '@types/node'
+      - less
+      - lightningcss
+      - sass
+      - stylus
+      - sugarss
+      - supports-color
+      - terser
+    optional: true
+
+  vite@5.2.14(@types/node@20.10.5)(terser@5.34.1):
     dependencies:
       esbuild: 0.20.2
       postcss: 8.4.47
@@ -24263,6 +24221,7 @@ snapshots:
     optionalDependencies:
       '@types/node': 20.10.5
       fsevents: 2.3.3
+      terser: 5.34.1
     optional: true
 
   vite@5.2.14(@types/node@20.11.29)(terser@5.34.1):
@@ -24275,10 +24234,21 @@ snapshots:
       fsevents: 2.3.3
       terser: 5.34.1
 
-  vitest@2.1.2(@types/node@20.10.5):
+  vite@5.2.14(@types/node@20.14.8)(terser@5.34.1):
+    dependencies:
+      esbuild: 0.20.2
+      postcss: 8.4.47
+      rollup: 4.13.0
+    optionalDependencies:
+      '@types/node': 20.14.8
+      fsevents: 2.3.3
+      terser: 5.34.1
+    optional: true
+
+  vitest@2.1.2(@types/node@20.10.5)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1):
     dependencies:
       '@vitest/expect': 2.1.2
-      '@vitest/mocker': 2.1.2(@vitest/spy@2.1.2)(vite@5.2.14(@types/node@20.10.5))
+      '@vitest/mocker': 2.1.2(@vitest/spy@2.1.2)(msw@2.4.9(typescript@5.4.5))(vite@5.2.14(@types/node@20.10.5)(terser@5.34.1))
       '@vitest/pretty-format': 2.1.2
       '@vitest/runner': 2.1.2
       '@vitest/snapshot': 2.1.2
@@ -24293,11 +24263,12 @@ snapshots:
       tinyexec: 0.3.0
       tinypool: 1.0.1
       tinyrainbow: 1.2.0
-      vite: 5.2.14(@types/node@20.10.5)
-      vite-node: 2.1.2(@types/node@20.10.5)
+      vite: 5.2.14(@types/node@20.10.5)(terser@5.34.1)
+      vite-node: 2.1.2(@types/node@20.10.5)(terser@5.34.1)
       why-is-node-running: 2.3.0
     optionalDependencies:
       '@types/node': 20.10.5
+      jsdom: 20.0.3
     transitivePeerDependencies:
       - less
       - lightningcss
@@ -24343,6 +24314,41 @@ snapshots:
       - supports-color
       - terser
 
+  vitest@2.1.2(@types/node@20.14.8)(jsdom@20.0.3)(msw@2.4.9(typescript@5.4.5))(terser@5.34.1):
+    dependencies:
+      '@vitest/expect': 2.1.2
+      '@vitest/mocker': 2.1.2(@vitest/spy@2.1.2)(msw@2.4.9(typescript@5.4.5))(vite@5.2.14(@types/node@20.14.8)(terser@5.34.1))
+      '@vitest/pretty-format': 2.1.2
+      '@vitest/runner': 2.1.2
+      '@vitest/snapshot': 2.1.2
+      '@vitest/spy': 2.1.2
+      '@vitest/utils': 2.1.2
+      chai: 5.1.1
+      debug: 4.3.7(supports-color@5.5.0)
+      magic-string: 0.30.11
+      pathe: 1.1.2
+      std-env: 3.7.0
+      tinybench: 2.9.0
+      tinyexec: 0.3.0
+      tinypool: 1.0.1
+      tinyrainbow: 1.2.0
+      vite: 5.2.14(@types/node@20.14.8)(terser@5.34.1)
+      vite-node: 2.1.2(@types/node@20.14.8)(terser@5.34.1)
+      why-is-node-running: 2.3.0
+    optionalDependencies:
+      '@types/node': 20.14.8
+      jsdom: 20.0.3
+    transitivePeerDependencies:
+      - less
+      - lightningcss
+      - msw
+      - sass
+      - stylus
+      - sugarss
+      - supports-color
+      - terser
+    optional: true
+
   vscode-jsonrpc@8.2.0: {}
 
   vscode-languageserver-protocol@3.17.5:

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.87.0",
+  "version": "2.89.0",
   "private": true,
   "license": "MIT",
   "engines": {

web/src/__tests__/ingestion.servertest.ts

@@ -10,6 +10,70 @@ import { cleanEvent } from "@langfuse/shared/src/server";
 describe("/api/public/ingestion API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
 
+  [
+    {
+      input: "Hello",
+      output: "World",
+    },
+    {
+      input: { key: "value" },
+      output: { key: "value" },
+    },
+    {
+      input: [1, 2, "hello"],
+      output: [3, 4, "world"],
+    },
+    // TODO: Enable test after prisma fixed https://github.com/prisma/prisma/issues/24989.
+    // {
+    //   input: "Test \ud83d",
+    //   output: "Result \ud83d",
+    // },
+  ].forEach((testConfig) => {
+    it(`should create generations with correct input and output ${JSON.stringify(
+      testConfig,
+    )}`, async () => {
+      const traceId = v4();
+      const generationId = v4();
+
+      const response = await makeAPICall("POST", "/api/public/ingestion", {
+        metadata: {
+          sdk_version: "1.0.0",
+          sdk_name: "python",
+        },
+        batch: [
+          {
+            id: v4(),
+            type: "observation-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: generationId,
+              traceId: traceId,
+              type: "GENERATION",
+              name: "generation-name",
+              startTime: "2021-01-01T00:00:00.000Z",
+              endTime: "2021-01-01T00:00:00.000Z",
+              modelParameters: { key: "value" },
+              input: testConfig.input,
+              output: testConfig.output,
+            },
+          },
+        ],
+      });
+
+      expect(response.status).toBe(207);
+
+      const dbGeneration = await prisma.observation.findUnique({
+        where: {
+          id: generationId,
+        },
+      });
+
+      expect(dbGeneration?.id).toBe(generationId);
+      expect(dbGeneration?.input).toStrictEqual(testConfig.input);
+      expect(dbGeneration?.output).toStrictEqual(testConfig.output);
+    });
+  });
+
   [
     {
       usage: {

web/src/__tests__/server/repositories/observation-repository.servertest.ts

@@ -42,7 +42,7 @@ describe("Clickhouse Observations Repository Test", () => {
       output: "sample_output",
       provided_model_name: "sample_model",
       internal_model_id: "sample_internal_model_id",
-      model_parameters: "sample_parameters",
+      model_parameters: '{"something":"sample_param"}',
       total_cost: 300,
       prompt_id: "sample_prompt_id",
       prompt_name: "sample_prompt_name",
@@ -69,7 +69,7 @@ describe("Clickhouse Observations Repository Test", () => {
     expect(result.input).toEqual(observation.input);
     expect(result.output).toEqual(observation.output);
     expect(result.internalModelId).toEqual(observation.internal_model_id);
-    expect(result.modelParameters).toEqual(observation.model_parameters);
+    expect(result.modelParameters).toEqual({ something: "sample_param" });
     expect(result.promptId).toEqual(observation.prompt_id);
     expect(result.endTime).toEqual(new Date(observation.end_time));
     expect(result.completionStartTime).toEqual(

web/src/components/app-sidebar.tsx

@@ -15,6 +15,7 @@ import { useRouter } from "next/router";
 import Link from "next/link";
 import { Alert, AlertDescription } from "@/src/components/ui/alert";
 import { LangfuseLogo } from "@/src/components/LangfuseLogo";
+import { SidebarNotifications } from "@/src/components/sidebar-notifications";
 
 type AppSidebarProps = {
   navItems: NavMainItem[];
@@ -34,12 +35,13 @@ export function AppSidebar({
         <div className="flex items-center gap-2 p-2">
           <LangfuseLogo version />
         </div>
+        <DemoBadge />
       </SidebarHeader>
       <SidebarContent>
         <NavMain items={navItems} />
         <div className="flex-1" />
-        <div className="p-2">
-          <DemoBadge />
+        <div className="flex flex-col gap-2 p-2">
+          <SidebarNotifications />
         </div>
         <NavMain items={secondaryNavItems} showFeedbackButton />
       </SidebarContent>
@@ -59,7 +61,7 @@ const DemoBadge = () => {
     env.NEXT_PUBLIC_DEMO_PROJECT_ID &&
     routerProjectId === env.NEXT_PUBLIC_DEMO_PROJECT_ID &&
     Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) ? (
-    <Alert className="bg-light-yellow group-data-[collapsible=icon]:hidden">
+    <Alert className="rounded-md bg-light-yellow group-data-[collapsible=icon]:hidden">
       <AlertDescription className="overflow-hidden text-ellipsis whitespace-nowrap text-xs">
         View-only{" "}
         <Link

web/src/components/grouped-score-badge.tsx

@@ -6,8 +6,7 @@ import {
 } from "@/src/components/ui/hover-card";
 import { MessageCircleMore } from "lucide-react";
 
-import { type APIScore } from "@langfuse/shared";
-import { type LastUserScore } from "@/src/features/scores/lib/types";
+import { type APIScore, type LastUserScore } from "@langfuse/shared";
 
 export const GroupedScoreBadges = <T extends APIScore | LastUserScore>({
   scores,

web/src/components/layouts/layout.tsx

@@ -7,10 +7,6 @@ import { env } from "@/src/env.mjs";
 import { Spinner } from "@/src/components/layouts/spinner";
 import { hasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import { Toaster } from "@/src/components/ui/sonner";
-import {
-  NOTIFICATIONS,
-  useCheckNotification,
-} from "@/src/features/notifications/checkNotifications";
 import DOMPurify from "dompurify";
 import { ThemeToggle } from "@/src/features/theming/ThemeToggle";
 import { useQueryProjectOrOrganization } from "@/src/features/projects/hooks";
@@ -106,8 +102,6 @@ export default function Layout(props: PropsWithChildren) {
     | undefined;
   const session = useSessionWithRetryOnUnauthenticated();
 
-  useCheckNotification(NOTIFICATIONS, session.status === "authenticated");
-
   const enableExperimentalFeatures =
     session.data?.environment.enableExperimentalFeatures ?? false;
 

web/src/components/scores-table-cell.tsx

@@ -6,7 +6,7 @@ import {
 import {
   type CategoricalAggregate,
   type NumericAggregate,
-} from "@/src/features/scores/lib/types";
+} from "@langfuse/shared";
 
 import { numberFormatter } from "@/src/utils/numbers";
 import { cn } from "@/src/utils/tailwind";

web/src/components/session/index.tsx

@@ -18,6 +18,7 @@ import { Button } from "@/src/components/ui/button";
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { CommentDrawerButton } from "@/src/features/comments/CommentDrawerButton";
 import { ScrollScreenPage } from "@/src/components/layouts/scroll-screen-page";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 // some projects have thousands of traces in a sessions, paginate to avoid rendering all at once
 const PAGE_SIZE = 50;
@@ -32,6 +33,7 @@ export const SessionPage: React.FC<{
     {
       sessionId,
       projectId: projectId,
+      queryClickhouse: useClickhouse(),
     },
     {
       retry(failureCount, error) {
@@ -44,7 +46,7 @@ export const SessionPage: React.FC<{
     if (session.isSuccess) {
       setDetailPageList(
         "traces",
-        session.data.traces.map((t) => t.id),
+        session.data.traces.map((t) => ({ id: t.id })),
       );
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -93,8 +95,8 @@ export const SessionPage: React.FC<{
           <DetailPageNav
             key="nav"
             currentId={encodeURIComponent(sessionId)}
-            path={(id) =>
-              `/project/${projectId}/sessions/${encodeURIComponent(id)}`
+            path={(entry) =>
+              `/project/${projectId}/sessions/${encodeURIComponent(entry.id)}`
             }
             listKey="sessions"
           />,
@@ -185,7 +187,7 @@ const SessionIO = ({
   projectId: string;
 }) => {
   const trace = api.traces.byId.useQuery(
-    { traceId, projectId },
+    { traceId, projectId, queryClickhouse: useClickhouse() },
     {
       enabled: typeof traceId === "string",
       trpc: {

web/src/components/sidebar-notifications.tsx

@@ -0,0 +1,125 @@
+import { Button } from "@/src/components/ui/button";
+import {
+  Card,
+  CardContent,
+  CardDescription,
+  CardHeader,
+  CardTitle,
+} from "@/src/components/ui/card";
+import { useEffect, useState } from "react";
+import { X } from "lucide-react";
+import useLocalStorage from "./useLocalStorage";
+import Link from "next/link";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+
+type SidebarNotification = {
+  id: string; // Add unique ID for each notification
+  title: string;
+  description: React.ReactNode;
+  link: string;
+  // defaults to "Learn more" if no linkContent and no linkTitle
+  linkTitle?: string;
+  linkContent?: React.ReactNode;
+};
+
+const notifications: SidebarNotification[] = [
+  {
+    id: "github-star",
+    title: "Star Langfuse",
+    description:
+      "See the latest releases and help grow the community on GitHub.",
+    link: "https://github.com/langfuse/langfuse",
+    linkContent: (
+      // eslint-disable-next-line @next/next/no-img-element
+      <img
+        alt="Langfuse Github stars"
+        src="https://img.shields.io/github/stars/langfuse/langfuse?label=langfuse&amp;style=social"
+      />
+    ),
+  },
+];
+
+const STORAGE_KEY = "dismissed-sidebar-notifications";
+
+export function SidebarNotifications() {
+  const capture = usePostHogClientCapture();
+  const [currentNotificationIndex, setCurrentNotificationIndex] = useState<
+    number | null
+  >(null);
+
+  const [dismissedNotifications, setDismissedNotifications] = useLocalStorage<
+    string[]
+  >(STORAGE_KEY, []);
+
+  // Find the first non-dismissed notification on mount or when dismissed list changes
+  useEffect(() => {
+    const firstAvailableIndex = notifications.findIndex(
+      (notif) => !dismissedNotifications.includes(notif.id),
+    );
+    setCurrentNotificationIndex(
+      firstAvailableIndex === -1 ? null : firstAvailableIndex,
+    );
+  }, [dismissedNotifications]);
+
+  const dismissNotification = (id: string) => {
+    setDismissedNotifications([...dismissedNotifications, id]);
+  };
+
+  if (currentNotificationIndex === null) {
+    return null;
+  }
+
+  const currentNotification = notifications[currentNotificationIndex];
+
+  return (
+    <Card className="relative overflow-hidden rounded-md bg-opacity-50 shadow-none group-data-[collapsible=icon]:hidden">
+      <Button
+        variant="ghost"
+        size="sm"
+        className="absolute right-2 top-2 h-6 w-6 p-0"
+        onClick={() => {
+          capture("notification:dismiss_notification", {
+            notification_id: currentNotification.id,
+          });
+          dismissNotification(currentNotification.id);
+        }}
+        title="Dismiss"
+      >
+        <X className="h-4 w-4" />
+      </Button>
+      <CardHeader className="p-4 pb-0">
+        <CardTitle className="text-sm">{currentNotification.title}</CardTitle>
+        <CardDescription>{currentNotification.description}</CardDescription>
+      </CardHeader>
+      <CardContent className="p-4 pt-2">
+        {currentNotification.linkContent ? (
+          <Link
+            href={currentNotification.link}
+            target="_blank"
+            onClick={() => {
+              capture("notification:click_link", {
+                notification_id: currentNotification.id,
+              });
+            }}
+          >
+            {currentNotification.linkContent}
+          </Link>
+        ) : (
+          <Button variant="secondary" size="sm" asChild>
+            <Link
+              href={currentNotification.link}
+              target="_blank"
+              onClick={() => {
+                capture("notification:click_link", {
+                  notification_id: currentNotification.id,
+                });
+              }}
+            >
+              {currentNotification.linkTitle ?? "Learn more"}
+            </Link>
+          </Button>
+        )}
+      </CardContent>
+    </Card>
+  );
+}

web/src/components/star-toggle.tsx

@@ -122,11 +122,13 @@ export function StarTraceToggle({
 export function StarTraceDetailsToggle({
   projectId,
   traceId,
+  timestamp,
   value,
   size = "icon",
 }: {
   projectId: string;
   traceId: string;
+  timestamp?: Date;
   value: boolean;
   size?: "icon" | "icon-xs";
 }) {
@@ -150,6 +152,7 @@ export function StarTraceDetailsToggle({
       const prevData = utils.traces.byIdWithObservationsAndScores.getData({
         traceId,
         projectId,
+        timestamp,
       });
 
       return { prevData };
@@ -159,7 +162,7 @@ export function StarTraceDetailsToggle({
       trpcErrorToast(err);
       // Rollback to the previous value if mutation fails
       utils.traces.byIdWithObservationsAndScores.setData(
-        { traceId, projectId },
+        { traceId, projectId, timestamp },
         context?.prevData,
       );
     },
@@ -167,7 +170,7 @@ export function StarTraceDetailsToggle({
       setIsLoading(false);
 
       utils.traces.byIdWithObservationsAndScores.setData(
-        { traceId, projectId },
+        { traceId, projectId, timestamp },
         (
           oldQueryData:
             | RouterOutput["traces"]["byIdWithObservationsAndScores"]

web/src/components/table/data-table.tsx

@@ -210,7 +210,7 @@ export function DataTable<TData extends object, TValue>({
           style={{ ...columnSizeVars }}
         >
           <Table>
-            <TableHeader>
+            <TableHeader className="sticky top-0 z-10">
               {tableHeaders.map((headerGroup) => (
                 <TableRow key={headerGroup.id}>
                   {headerGroup.headers.map((header) => {

web/src/components/table/use-cases/generations.tsx

@@ -41,7 +41,7 @@ import {
 } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
 import { useTableDateRange } from "@/src/hooks/useTableDateRange";
 import { useDebounce } from "@/src/hooks/useDebounce";
-import { type ScoreAggregate } from "@/src/features/scores/lib/types";
+import { type ScoreAggregate } from "@langfuse/shared";
 import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns";
 import TagList from "@/src/features/tag/components/TagList";
 import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";

web/src/components/table/use-cases/scores.tsx

@@ -27,6 +27,7 @@ import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 import TagList from "@/src/features/tag/components/TagList";
 import { cn } from "@/src/utils/tailwind";
 import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 export type ScoresTableRow = {
   id: string;
@@ -131,6 +132,7 @@ export default function ScoresTable({
     page: 0,
     limit: 1,
     orderBy: null,
+    queryClickhouse: useClickhouse(),
   };
 
   const getAllPayload = {
@@ -138,6 +140,7 @@ export default function ScoresTable({
     page: paginationState.pageIndex,
     limit: paginationState.pageSize,
     orderBy: orderByState,
+    queryClickhouse: useClickhouse(),
   };
 
   const scores = api.scores.all.useQuery(getAllPayload);
@@ -151,6 +154,7 @@ export default function ScoresTable({
         dateRangeFilter[0]?.type === "datetime"
           ? dateRangeFilter[0]
           : undefined,
+      queryClickhouse: useClickhouse(),
     },
     {
       trpc: {
@@ -332,7 +336,7 @@ export default function ScoresTable({
         href: "https://langfuse.com/docs/scores/model-based-evals",
       },
       enableHiding: true,
-      enableSorting: true,
+      enableSorting: false,
       size: 150,
       cell: ({ row }) => {
         const value = row.getValue("jobConfigurationId");

web/src/components/table/use-cases/sessions.tsx

@@ -29,6 +29,7 @@ import TagList from "@/src/features/tag/components/TagList";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { cn } from "@/src/utils/tailwind";
 import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 export type SessionTableRow = {
   id: string;
@@ -109,6 +110,7 @@ export default function SessionsTable({
     orderBy: null,
     page: 0,
     limit: 1,
+    queryClickhouse: useClickhouse(),
   };
 
   const payloadGetAll = {
@@ -125,6 +127,7 @@ export default function SessionsTable({
     {
       projectId,
       sessionIds: sessions.data?.sessions.map((s) => s.id) ?? [],
+      queryClickhouse: useClickhouse(),
     },
     {
       enabled: sessions.data !== undefined,
@@ -146,6 +149,7 @@ export default function SessionsTable({
         dateRangeFilter[0]?.type === "datetime"
           ? dateRangeFilter[0]
           : undefined,
+      queryClickhouse: useClickhouse(),
     },
     {
       trpc: {
@@ -165,7 +169,7 @@ export default function SessionsTable({
     if (sessions.isSuccess) {
       setDetailPageList(
         "sessions",
-        sessions.data.sessions.map((t) => t.id),
+        sessions.data.sessions.map((t) => ({ id: t.id })),
       );
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -499,21 +503,23 @@ export default function SessionsTable({
               : {
                   isLoading: false,
                   isError: false,
-                  data: sessionRowData.rows?.map((session) => ({
-                    id: session.id,
-                    createdAt: session.createdAt.toLocaleString(),
-                    userIds: session.userIds,
-                    countTraces: session.countTraces,
-                    bookmarked: session.bookmarked,
-                    sessionDuration: session.sessionDuration,
-                    inputCost: session.inputCost,
-                    outputCost: session.outputCost,
-                    totalCost: session.totalCost,
-                    inputTokens: session.promptTokens,
-                    outputTokens: session.completionTokens,
-                    totalTokens: session.totalTokens,
-                    traceTags: session.traceTags,
-                  })),
+                  data: sessionRowData.rows?.map<SessionTableRow>(
+                    (session) => ({
+                      id: session.id,
+                      bookmarked: session.bookmarked,
+                      createdAt: session.createdAt.toLocaleString(),
+                      userIds: session.userIds,
+                      countTraces: session.countTraces,
+                      sessionDuration: session.sessionDuration,
+                      inputCost: session.inputCost,
+                      outputCost: session.outputCost,
+                      totalCost: session.totalCost,
+                      inputTokens: session.promptTokens,
+                      outputTokens: session.completionTokens,
+                      totalTokens: session.totalTokens,
+                      traceTags: session.traceTags,
+                    }),
+                  ),
                 }
         }
         pagination={{

web/src/components/table/use-cases/traces.tsx

@@ -43,7 +43,7 @@ import {
 } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
 import { useTableDateRange } from "@/src/hooks/useTableDateRange";
 import { useDebounce } from "@/src/hooks/useDebounce";
-import { type ScoreAggregate } from "@/src/features/scores/lib/types";
+import { type ScoreAggregate } from "@langfuse/shared";
 import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns";
 import { joinTableCoreAndMetrics } from "@/src/components/table/utils/joinTableCoreAndMetrics";
 import { Skeleton } from "@/src/components/ui/skeleton";
@@ -54,7 +54,7 @@ import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 export type TracesTableRow = {
   bookmarked: boolean;
   id: string;
-  timestamp: string;
+  timestamp: Date;
   name: string;
   userId: string;
   level?: ObservationLevel;
@@ -163,6 +163,7 @@ export default function TracesTable({
   const traceMetrics = api.traces.metrics.useQuery(
     {
       projectId,
+      filter: filterState,
       traceIds: traces.data?.traces.map((t) => t.id) ?? [],
       queryClickhouse: useClickhouse(),
     },
@@ -185,7 +186,10 @@ export default function TracesTable({
     if (traces.isSuccess) {
       setDetailPageList(
         "traces",
-        traces.data.traces.map((t) => t.id),
+        traces.data.traces.map((t) => ({
+          id: t.id,
+          params: { timestamp: t.timestamp.toISOString() },
+        })),
       );
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -240,7 +244,7 @@ export default function TracesTable({
       size: 30,
       isPinned: true,
       header: ({ table }) => (
-        <div className="mt-1 h-5">
+        <div className="flex h-full items-center">
           <Checkbox
             checked={
               table.getIsAllPageRowsSelected()
@@ -300,9 +304,12 @@ export default function TracesTable({
       isPinned: true,
       cell: ({ row }) => {
         const value: TracesTableRow["id"] = row.getValue("id");
+        const timestamp: TracesTableRow["timestamp"] =
+          row.getValue("timestamp");
+
         return value && typeof value === "string" ? (
           <TableLink
-            path={`/project/${projectId}/traces/${encodeURIComponent(value)}`}
+            path={`/project/${projectId}/traces/${encodeURIComponent(value)}?timestamp=${encodeURIComponent(timestamp.toISOString())}`}
             value={value}
           />
         ) : undefined;
@@ -316,6 +323,10 @@ export default function TracesTable({
       size: 150,
       enableHiding: true,
       enableSorting: true,
+      cell: ({ row }) => {
+        const value: TracesTableRow["timestamp"] = row.getValue("timestamp");
+        return value ? new Date(value).toLocaleString() : undefined;
+      },
     },
     {
       accessorKey: "name",
@@ -520,10 +531,13 @@ export default function TracesTable({
       size: 400,
       cell: ({ row }) => {
         const traceId: TracesTableRow["id"] = row.getValue("id");
+        const traceTimestamp: TracesTableRow["timestamp"] =
+          row.getValue("timestamp");
         return (
           <TracesDynamicCell
             traceId={traceId}
             projectId={projectId}
+            timestamp={new Date(traceTimestamp)}
             col="input"
             singleLine={rowHeight === "s"}
           />
@@ -539,10 +553,13 @@ export default function TracesTable({
       size: 400,
       cell: ({ row }) => {
         const traceId: TracesTableRow["id"] = row.getValue("id");
+        const traceTimestamp: TracesTableRow["timestamp"] =
+          row.getValue("timestamp");
         return (
           <TracesDynamicCell
             traceId={traceId}
             projectId={projectId}
+            timestamp={new Date(traceTimestamp)}
             col="output"
             singleLine={rowHeight === "s"}
           />
@@ -561,10 +578,13 @@ export default function TracesTable({
       },
       cell: ({ row }) => {
         const traceId: TracesTableRow["id"] = row.getValue("id");
+        const traceTimestamp: TracesTableRow["timestamp"] =
+          row.getValue("timestamp");
         return (
           <TracesDynamicCell
             traceId={traceId}
             projectId={projectId}
+            timestamp={new Date(traceTimestamp)}
             col="metadata"
             singleLine={rowHeight === "s"}
           />
@@ -707,7 +727,7 @@ export default function TracesTable({
           return {
             bookmarked: trace.bookmarked,
             id: trace.id,
-            timestamp: trace.timestamp.toLocaleString(),
+            timestamp: trace.timestamp,
             name: trace.name ?? "",
             level: trace.level,
             observationCount: trace.observationCount,
@@ -817,16 +837,18 @@ export default function TracesTable({
 const TracesDynamicCell = ({
   traceId,
   projectId,
+  timestamp,
   col,
   singleLine = false,
 }: {
   traceId: string;
   projectId: string;
+  timestamp: Date;
   col: "input" | "output" | "metadata";
   singleLine?: boolean;
 }) => {
   const trace = api.traces.byId.useQuery(
-    { traceId, projectId, queryClickhouse: useClickhouse() },
+    { traceId, projectId, queryClickhouse: useClickhouse(), timestamp },
     {
       enabled: typeof traceId === "string",
       trpc: {

web/src/components/trace/IOPreview.tsx

@@ -26,7 +26,7 @@ const isSupportedMarkdownFormat = (
 
 // MarkdownOrJsonView will render markdown if `isMarkdownEnabled` (global context) is true and the content is valid markdown
 // otherwise, if content is valid markdown will render JSON with switch to enable markdown globally
-function MarkdownOrJsonView({
+export function MarkdownOrJsonView({
   content,
   title,
   className,

web/src/components/trace/ObservationPreview.tsx

@@ -289,6 +289,7 @@ export const ObservationPreview = ({
           {selectedTab === "scores" && (
             <ScoresTable
               projectId={projectId}
+              traceId={traceId}
               omittedFilter={["Observation ID"]}
               observationId={preloadedObservation.id}
               hiddenColumns={[

web/src/components/trace/index.tsx

@@ -246,7 +246,13 @@ export function Trace(props: {
   );
 }
 
-export function TracePage({ traceId }: { traceId: string }) {
+export function TracePage({
+  traceId,
+  timestamp,
+}: {
+  traceId: string;
+  timestamp?: Date;
+}) {
   const capture = usePostHogClientCapture();
   const router = useRouter();
   const utils = api.useUtils();
@@ -256,6 +262,7 @@ export function TracePage({ traceId }: { traceId: string }) {
   const trace = api.traces.byIdWithObservationsAndScores.useQuery(
     {
       traceId,
+      timestamp,
       projectId: router.query.projectId as string,
       queryClickhouse: useClickhouse(),
     },
@@ -328,7 +335,7 @@ export function TracePage({ traceId }: { traceId: string }) {
             />
             <DetailPageNav
               currentId={traceId}
-              path={(id) => {
+              path={(entry) => {
                 const { view, display, projectId } = router.query;
                 const queryParams = new URLSearchParams({
                   ...(typeof view === "string" ? { view } : {}),
@@ -337,7 +344,13 @@ export function TracePage({ traceId }: { traceId: string }) {
                 const queryParamString = Boolean(queryParams.size)
                   ? `?${queryParams.toString()}`
                   : "";
-                return `/project/${projectId as string}/traces/${id}${queryParamString}`;
+
+                const timestamp =
+                  entry.params && entry.params.timestamp
+                    ? encodeURIComponent(entry.params.timestamp)
+                    : undefined;
+
+                return `/project/${projectId as string}/traces/${entry.id}${queryParamString}${timestamp ? `?timestamp=${timestamp}` : ""}`;
               }}
               listKey="traces"
             />

web/src/components/ui/table.tsx

@@ -74,7 +74,7 @@ const TableHead = React.forwardRef<
   <th
     ref={ref}
     className={cn(
-      "sticky top-0 z-10 border-b bg-background px-4 text-left align-middle font-medium text-muted-foreground [&:has([role=checkbox])]:pr-0",
+      "relative h-10 border-b bg-background px-4 text-left align-middle font-medium text-muted-foreground [&:has([role=checkbox])]:pr-0",
       className,
     )}
     {...props}

web/src/components/ui/time-picker-input.tsx

@@ -104,7 +104,7 @@ const TimePickerInput = React.forwardRef<
         id={id || picker}
         name={name || picker}
         className={cn(
-          "size-7 p-0 text-center tabular-nums caret-transparent focus:bg-accent focus:text-accent-foreground [&::-webkit-inner-spin-button]:appearance-none",
+          "size-7 min-w-0 p-0 text-center tabular-nums caret-transparent focus:bg-accent focus:text-accent-foreground [&::-webkit-inner-spin-button]:appearance-none",
           className,
         )}
         value={value || calculatedValue}

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.87.0";
+export const VERSION = "v2.89.0";

web/src/ee/features/annotation-queues/components/AnnotationQueueItemsTable.tsx

@@ -167,7 +167,7 @@ export function AnnotationQueueItemsTable({
       isPinned: true,
       header: ({ table }) => {
         return (
-          <div className="mt-1 h-5">
+          <div className="flex h-full items-center">
             <Checkbox
               checked={
                 table.getIsAllPageRowsSelected()

web/src/ee/features/annotation-queues/components/CreateOrEditAnnotationQueueButton.tsx

@@ -277,7 +277,7 @@ export const CreateOrEditAnnotationQueueButton = ({
                     </FormDescription>
                     <FormControl>
                       <MultiSelectKeyValues
-                        title="Value"
+                        placeholder="Value"
                         align="end"
                         className="grid grid-cols-[auto,1fr,auto,auto] gap-2"
                         onValueChange={handleOnValueChange}

web/src/ee/features/billing/components/UsageTracker.tsx

@@ -28,6 +28,7 @@ export const UsageTracker = () => {
       refetchOnMount: false,
       refetchOnReconnect: false,
       refetchOnWindowFocus: false,
+      staleTime: 60 * 60 * 1000,
     },
   );
 

web/src/ee/features/billing/server/cloudBillingRouter.ts

@@ -106,6 +106,9 @@ export const cloudBillingRouter = createTRPCRouter({
         tax_id_collection: {
           enabled: true,
         },
+        automatic_tax: {
+          enabled: true,
+        },
         consent_collection: {
           terms_of_service: "required",
         },

web/src/ee/features/evals/components/evaluator-detail.tsx

@@ -65,7 +65,7 @@ export const EvaluatorDetail = () => {
     <FullScreenPage>
       <>
         <Header
-          title={`${evaluator.data?.id}` ?? "Loading..."}
+          title={evaluator.data?.id ?? "Loading..."}
           breadcrumb={[
             {
               name: "Evaluators",
@@ -84,8 +84,8 @@ export const EvaluatorDetail = () => {
                 <DetailPageNav
                   key="nav"
                   currentId={encodeURIComponent(evaluator.data.id)}
-                  path={(id) =>
-                    `/project/${projectId}/evals/${encodeURIComponent(id)}`
+                  path={(entry) =>
+                    `/project/${projectId}/evals/${encodeURIComponent(entry.id)}`
                   }
                   listKey="evals"
                 />
@@ -117,10 +117,7 @@ export const EvaluatorDetail = () => {
                   <span className="text-sm font-medium">Eval Template</span>
                   <TableLink
                     path={`/project/${projectId}/evals/templates/${existingEvaluator.evalTemplateId}`}
-                    value={
-                      `${existingEvaluator.evalTemplate.name} (v${existingEvaluator.evalTemplate.version})` ??
-                      ""
-                    }
+                    value={`${existingEvaluator.evalTemplate.name} (v${existingEvaluator.evalTemplate.version})`}
                     className="flex min-h-6 items-center"
                   />
                 </CardDescription>

web/src/ee/features/evals/components/evaluator-table.tsx

@@ -50,7 +50,7 @@ export default function EvaluatorTable({
     if (evaluators.isSuccess) {
       setDetailPageList(
         "evals",
-        evaluators.data.configs.map((evaluator) => evaluator.id),
+        evaluators.data.configs.map((evaluator) => ({ id: evaluator.id })),
       );
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps

web/src/env.mjs

@@ -47,6 +47,8 @@ export const env = createEnv({
       .enum(["OWNER", "ADMIN", "MEMBER", "VIEWER"])
       .optional(),
     LANGFUSE_CSP_ENFORCE_HTTPS: z.enum(["true", "false"]).optional(),
+    // Telemetry
+    TELEMETRY_ENABLED: z.enum(["true", "false"]).optional(),
     // AUTH
     AUTH_GOOGLE_CLIENT_ID: z.string().optional(),
     AUTH_GOOGLE_CLIENT_SECRET: z.string().optional(),
@@ -185,10 +187,15 @@ export const env = createEnv({
           (creator) => emailSchema.safeParse(creator).success,
         );
       }, "LANGFUSE_ALLOWED_ORGANIZATION_CREATORS must be a comma separated list of valid email addresses"),
-    LANGFUSE_INGESTION_QUEUE_DELAY_SECONDS: z.coerce
+    LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
       .number()
       .nonnegative()
-      .default(0),
+      .default(15_000),
+    LANGFUSE_READ_FROM_POSTGRES_ONLY: z.enum(["true", "false"]).default("true"),
+    LANGFUSE_RETURN_FROM_CLICKHOUSE: z.enum(["true", "false"]).default("false"),
+    LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE: z
+      .enum(["true", "false"])
+      .default("false"),
     STRIPE_SECRET_KEY: z.string().optional(),
     STRIPE_WEBHOOK_SIGNING_SECRET: z.string().optional(),
     SENTRY_AUTH_TOKEN: z.string().optional(),
@@ -258,6 +265,7 @@ export const env = createEnv({
       process.env.LANGFUSE_NEW_USER_SIGNUP_WEBHOOK,
     SALT: process.env.SALT,
     LANGFUSE_CSP_ENFORCE_HTTPS: process.env.LANGFUSE_CSP_ENFORCE_HTTPS,
+    TELEMETRY_ENABLED: process.env.TELEMETRY_ENABLED,
     // Default org, project and role
     LANGFUSE_DEFAULT_ORG_ID: process.env.LANGFUSE_DEFAULT_ORG_ID,
     LANGFUSE_DEFAULT_ORG_ROLE: process.env.LANGFUSE_DEFAULT_ORG_ROLE,
@@ -390,8 +398,14 @@ export const env = createEnv({
       process.env.LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING,
     LANGFUSE_ALLOWED_ORGANIZATION_CREATORS:
       process.env.LANGFUSE_ALLOWED_ORGANIZATION_CREATORS,
-    LANGFUSE_INGESTION_QUEUE_DELAY_SECONDS:
-      process.env.LANGFUSE_INGESTION_QUEUE_DELAY_SECONDS,
+    LANGFUSE_INGESTION_QUEUE_DELAY_MS:
+      process.env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
+    LANGFUSE_READ_FROM_POSTGRES_ONLY:
+      process.env.LANGFUSE_READ_FROM_POSTGRES_ONLY,
+    LANGFUSE_RETURN_FROM_CLICKHOUSE:
+      process.env.LANGFUSE_RETURN_FROM_CLICKHOUSE,
+    LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE:
+      process.env.LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE,
     STRIPE_SECRET_KEY: process.env.STRIPE_SECRET_KEY,
     STRIPE_WEBHOOK_SIGNING_SECRET: process.env.STRIPE_WEBHOOK_SIGNING_SECRET,
     SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,

web/src/features/dashboard/components/ChartScores.tsx

@@ -8,7 +8,7 @@ import {
   fillMissingValuesAndTransform,
   isEmptyTimeSeries,
 } from "@/src/features/dashboard/components/hooks";
-
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 import { createTracesTimeFilter } from "@/src/features/dashboard/lib/dashboard-utils";
 import {
   dashboardDateRangeAggregationSettings,
@@ -56,6 +56,8 @@ export function ChartScores(props: {
         { type: "string", column: "scoreDataType" },
         { type: "string", column: "scoreSource" },
       ],
+      queryClickhouse: useClickhouse(),
+      queryName: "scores-aggregate-timeseries",
     },
     {
       trpc: {

web/src/features/dashboard/components/LatencyChart.tsx

@@ -15,6 +15,7 @@ import {
   type DashboardDateRangeAggregationOption,
 } from "@/src/utils/date-range-utils";
 import { NoDataOrLoading } from "@/src/components/NoDataOrLoading";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 export const GenerationLatencyChart = ({
   className,
@@ -56,6 +57,8 @@ export const GenerationLatencyChart = ({
         },
         { type: "string", column: "model" },
       ],
+      queryClickhouse: useClickhouse(),
+      queryName: "model-latencies-over-time",
     },
     {
       trpc: {
@@ -66,7 +69,7 @@ export const GenerationLatencyChart = ({
     },
   );
 
-  const allModels = getAllModels(projectId, globalFilterState);
+  const allModels = getAllModels(projectId, globalFilterState, useClickhouse());
 
   const getData = (valueColumn: string) => {
     return latencies.data && allModels.length > 0

web/src/features/dashboard/components/LatencyTables.tsx

@@ -9,6 +9,7 @@ import { formatIntervalSeconds } from "@/src/utils/dates";
 import { createTracesTimeFilter } from "@/src/features/dashboard/lib/dashboard-utils";
 import { truncate } from "@/src/utils/string";
 import { Popup } from "@/src/components/layouts/doc-popup";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 export const LatencyTables = ({
   projectId,
@@ -41,6 +42,8 @@ export const LatencyTables = ({
       orderBy: [
         { column: "duration", agg: "95thPercentile", direction: "DESC" },
       ],
+      queryClickhouse: useClickhouse(),
+      queryName: "observation-latencies-aggregated",
     },
     {
       trpc: {
@@ -75,6 +78,8 @@ export const LatencyTables = ({
       orderBy: [
         { column: "duration", agg: "95thPercentile", direction: "DESC" },
       ],
+      queryClickhouse: useClickhouse(),
+      queryName: "observation-latencies-aggregated",
     },
     {
       trpc: {
@@ -101,6 +106,8 @@ export const LatencyTables = ({
       orderBy: [
         { column: "duration", agg: "95thPercentile", direction: "DESC" },
       ],
+      queryClickhouse: useClickhouse(),
+      queryName: "traces-latencies-aggregated",
     },
     {
       trpc: {

web/src/features/dashboard/components/ModelCostTable.tsx

@@ -10,6 +10,7 @@ import { TotalMetric } from "./TotalMetric";
 import { totalCostDashboardFormatted } from "@/src/features/dashboard/lib/dashboard-utils";
 import { env } from "@/src/env.mjs";
 import { truncate } from "@/src/utils/string";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 export const ModelCostTable = ({
   className,
@@ -44,6 +45,8 @@ export const ModelCostTable = ({
       orderBy: [
         { column: "calculatedTotalCost", direction: "DESC", agg: "SUM" },
       ],
+      queryClickhouse: useClickhouse(),
+      queryName: "observations-model-cost",
     },
     {
       trpc: {

web/src/features/dashboard/components/ModelUsageChart.tsx

@@ -18,7 +18,7 @@ import {
   dashboardDateRangeAggregationSettings,
   type DashboardDateRangeAggregationOption,
 } from "@/src/utils/date-range-utils";
-
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 import { env } from "@/src/env.mjs";
 import { NoDataOrLoading } from "@/src/components/NoDataOrLoading";
 
@@ -62,6 +62,8 @@ export const ModelUsageChart = ({
       orderBy: [
         { column: "calculatedTotalCost", direction: "DESC", agg: "SUM" },
       ],
+      queryClickhouse: useClickhouse(),
+      queryName: "observations-usage-timeseries",
     },
     {
       trpc: {
@@ -72,7 +74,7 @@ export const ModelUsageChart = ({
     },
   );
 
-  const allModels = getAllModels(projectId, globalFilterState);
+  const allModels = getAllModels(projectId, globalFilterState, useClickhouse());
 
   const transformedTotalTokens =
     tokens.data && allModels.length > 0

web/src/features/dashboard/components/ScoresTable.tsx

@@ -15,6 +15,7 @@ import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDeta
 import { isCategoricalDataType } from "@/src/features/scores/lib/helpers";
 import { type DatabaseRow } from "@/src/server/api/services/queryBuilder";
 import { NoDataOrLoading } from "@/src/components/NoDataOrLoading";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 const dropValuesForCategoricalScores = (
   value: number,
@@ -45,6 +46,8 @@ export const ScoresTable = ({
     globalFilterState,
     "scoreTimestamp",
   );
+
+  const useCh = useClickhouse();
   const metrics = api.dashboard.chart.useQuery(
     {
       projectId,
@@ -69,6 +72,8 @@ export const ScoresTable = ({
         },
       ],
       orderBy: [{ column: "scoreId", direction: "DESC", agg: "COUNT" }],
+      queryClickhouse: useCh,
+      queryName: "score-aggregate",
     },
     {
       trpc: {
@@ -111,6 +116,8 @@ export const ScoresTable = ({
           },
         ],
         orderBy: [{ column: "scoreId", direction: "DESC", agg: "COUNT" }],
+        queryClickhouse: useCh,
+        queryName: "score-aggregate",
       },
       {
         trpc: {

web/src/features/dashboard/components/TracesBarListChart.tsx

@@ -7,6 +7,7 @@ import { TotalMetric } from "@/src/features/dashboard/components/TotalMetric";
 import { BarList } from "@tremor/react";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { NoDataOrLoading } from "@/src/components/NoDataOrLoading";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 export const TracesBarListChart = ({
   className,
@@ -28,6 +29,8 @@ export const TracesBarListChart = ({
       from: "traces",
       select: [{ column: "traceId", agg: "COUNT" }],
       filter: timeFilter,
+      queryClickhouse: useClickhouse(),
+      queryName: "traces-total",
     },
     {
       trpc: {
@@ -46,6 +49,8 @@ export const TracesBarListChart = ({
       filter: timeFilter,
       groupBy: [{ column: "traceName", type: "string" }],
       orderBy: [{ column: "traceId", direction: "DESC", agg: "COUNT" }],
+      queryClickhouse: useClickhouse(),
+      queryName: "traces-grouped-by-name",
     },
     {
       trpc: {

web/src/features/dashboard/components/TracesTimeSeriesChart.tsx

@@ -10,6 +10,7 @@ import {
   type DashboardDateRangeAggregationOption,
 } from "@/src/utils/date-range-utils";
 import { NoDataOrLoading } from "@/src/components/NoDataOrLoading";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 export const TracesTimeSeriesChart = ({
   className,
@@ -37,6 +38,8 @@ export const TracesTimeSeriesChart = ({
           temporalUnit: dashboardDateRangeAggregationSettings[agg].date_trunc,
         },
       ],
+      queryClickhouse: useClickhouse(),
+      queryName: "traces-timeseries",
     },
     {
       trpc: {

web/src/features/dashboard/components/UserChart.tsx

@@ -14,6 +14,7 @@ import {
 import { env } from "@/src/env.mjs";
 import { type DashboardDateRangeAggregationOption } from "@/src/utils/date-range-utils";
 import { NoDataOrLoading } from "@/src/components/NoDataOrLoading";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 
 type BarChartDataPoint = {
   name: string;
@@ -60,6 +61,8 @@ export const UserChart = ({
       orderBy: [
         { column: "calculatedTotalCost", direction: "DESC", agg: "SUM" },
       ],
+      queryClickhouse: useClickhouse(),
+      queryName: "observations-usage-by-users",
     },
     {
       trpc: {
@@ -83,6 +86,8 @@ export const UserChart = ({
         },
       ],
       orderBy: [{ column: "traceId", agg: "COUNT", direction: "DESC" }],
+      queryClickhouse: useClickhouse(),
+      queryName: "traces-grouped-by-user",
     },
     {
       trpc: {

web/src/features/dashboard/components/hooks.ts

@@ -6,6 +6,7 @@ import { api } from "@/src/utils/api";
 export const getAllModels = (
   projectId: string,
   globalFilterState: FilterState,
+  useClickhouse: boolean,
 ) => {
   const allModels = api.dashboard.chart.useQuery(
     {
@@ -22,6 +23,8 @@ export const getAllModels = (
         },
       ],
       groupBy: [{ type: "string", column: "model" }],
+      queryClickhouse: useClickhouse,
+      queryName: "distinct-models",
     },
     {
       trpc: {

web/src/features/dashboard/components/score-analytics/ScoreAnalytics.tsx

@@ -75,7 +75,7 @@ export function ScoreAnalytics(props: {
         !scoreKeysAndProps.isLoading &&
         Boolean(scoreKeysAndProps.data?.length) && (
           <MultiSelectKeyValues
-            title="Search score..."
+            placeholder="Search score..."
             onValueChange={(values, changedValueId, selectedValueKeys) => {
               if (values.length === 0) setSelectedDashboardScoreKeys([]);
 

web/src/features/dashboard/server/dashboard-router.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-
+import { isClickhouseAdminEligible } from "@/src/server/utils/checkClickhouseAccess";
 import {
   createTRPCRouter,
   protectedProjectProcedure,
@@ -7,9 +7,29 @@ import {
 import { executeQuery } from "@/src/server/api/services/queryBuilder";
 import {
   filterInterface,
+  type groupByInterface,
   sqlInterface,
 } from "@/src/server/api/services/sqlInterface";
 import { createHistogramData } from "@/src/features/dashboard/lib/score-analytics-utils";
+import { TRPCError } from "@trpc/server";
+import {
+  getTotalTraces,
+  getTracesGroupedByName,
+  getObservationsCostGroupedByName,
+  getScoreAggregate,
+  getObservationUsageByTime,
+  groupTracesByTime,
+  getDistinctModels,
+  getScoresAggregateOverTime,
+  getTracesGroupedByUsers,
+  getModelUsageByUser,
+  getModelLatenciesOverTime,
+  getObservationLatencies,
+  getTracesLatencies,
+} from "@langfuse/shared/src/server";
+import { type DatabaseRow } from "@/src/server/api/services/queryBuilder";
+import { dashboardColumnDefinitions } from "@langfuse/shared";
+import { env } from "@/src/env.mjs";
 
 export const dashboardRouter = createTRPCRouter({
   chart: protectedProjectProcedure
@@ -17,9 +37,214 @@ export const dashboardRouter = createTRPCRouter({
       sqlInterface.extend({
         projectId: z.string(),
         filter: filterInterface.optional(),
+        queryClickhouse: z.boolean().default(false),
+        queryName: z
+          .enum([
+            "traces-total",
+            "traces-grouped-by-name",
+            "observations-model-cost",
+            "score-aggregate",
+            "traces-timeseries",
+            "observations-usage-timeseries",
+            "distinct-models",
+            "scores-aggregate-timeseries",
+            "observations-usage-by-users",
+            "traces-grouped-by-user",
+            "observation-latencies-aggregated",
+            "traces-latencies-aggregated",
+            "model-latencies-over-time",
+          ])
+          .nullish(),
       }),
     )
     .query(async ({ input, ctx }) => {
+      if (
+        input.queryClickhouse &&
+        !isClickhouseAdminEligible(ctx.session.user)
+      ) {
+        throw new TRPCError({
+          code: "UNAUTHORIZED",
+          message: "Not eligible to query clickhouse",
+        });
+      }
+
+      if (
+        input.queryClickhouse ||
+        env.LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE === "true"
+      ) {
+        switch (input.queryName) {
+          case "traces-total":
+            const count = await getTotalTraces(
+              input.projectId,
+              input.filter ?? [],
+            );
+            return count as DatabaseRow[];
+          case "traces-grouped-by-name":
+            return (
+              await getTracesGroupedByName(
+                input.projectId,
+                dashboardColumnDefinitions,
+                input.filter,
+              )
+            ).map(
+              (row) =>
+                ({
+                  traceName: row.name,
+                  countTraceId: row.count,
+                }) as DatabaseRow,
+            );
+          case "observations-model-cost":
+            const cost = await getObservationsCostGroupedByName(
+              input.projectId,
+              input.filter ?? [],
+            );
+
+            return cost.map((row) => ({
+              model: row.name,
+              sumCalculatedTotalCost: row.sum_cost_details,
+              sumTotalTokens: row.sum_usage_details,
+            })) as DatabaseRow[];
+          case "score-aggregate":
+            const scores = await getScoreAggregate(
+              input.projectId,
+              input.filter ?? [],
+            );
+
+            return scores.map((row) => ({
+              scoreName: row.name,
+              scoreSource: row.source,
+              scoreDataType: row.data_type,
+              avgValue: row.avg_value,
+              countScoreId: Number(row.count),
+            })) as DatabaseRow[];
+
+          case "traces-timeseries":
+            const dateTrunc = extractTimeSeries(input.groupBy);
+            if (!dateTrunc) {
+              return [];
+            }
+            const rows = await groupTracesByTime(
+              input.projectId,
+              input.filter ?? [],
+              dateTrunc,
+            );
+
+            return rows as DatabaseRow[];
+          case "observations-usage-timeseries":
+            const dateTruncObs = extractTimeSeries(input.groupBy);
+            if (!dateTruncObs) {
+              return [];
+            }
+            const rowsObs = await getObservationUsageByTime(
+              input.projectId,
+              input.filter ?? [],
+              dateTruncObs,
+            );
+
+            return rowsObs.map((row) => ({
+              startTime: row.start_time,
+              sumTotalTokens: row.sum_usage_details,
+              sumCalculatedTotalCost: row.sum_cost_details,
+              model: row.provided_model_name,
+            })) as DatabaseRow[];
+
+          case "distinct-models":
+            const models = await getDistinctModels(
+              input.projectId,
+              input.filter ?? [],
+            );
+            return models as DatabaseRow[];
+
+          case "scores-aggregate-timeseries":
+            const dateTruncScores = extractTimeSeries(input.groupBy);
+            if (!dateTruncScores) {
+              return [];
+            }
+            const aggregatedScores = await getScoresAggregateOverTime(
+              input.projectId,
+              input.filter ?? [],
+              dateTruncScores,
+            );
+
+            return aggregatedScores as DatabaseRow[];
+
+          case "observations-usage-by-users":
+            const rowsUsers = await getModelUsageByUser(
+              input.projectId,
+              input.filter ?? [],
+            );
+
+            return rowsUsers.map((row) => ({
+              sumTotalTokens: row.sumUsageDetails,
+              sumCalculatedTotalCost: row.sumCostDetails,
+              user: row.userId,
+            })) as DatabaseRow[];
+
+          case "traces-grouped-by-user":
+            const traces = await getTracesGroupedByUsers(
+              input.projectId,
+              input.filter ?? [],
+              dashboardColumnDefinitions,
+            );
+
+            return traces.map((row) => ({
+              user: row.user,
+              countTraceId: Number(row.count),
+            })) as DatabaseRow[];
+          case "observation-latencies-aggregated":
+            const latencies = await getObservationLatencies(
+              input.projectId,
+              input.filter ?? [],
+            );
+
+            return latencies.map((row) => ({
+              name: row.name,
+              percentile50Duration: row.p50,
+              percentile90Duration: row.p90,
+              percentile95Duration: row.p95,
+              percentile99Duration: row.p99,
+            })) as DatabaseRow[];
+          case "model-latencies-over-time":
+            const dateTruncModels = extractTimeSeries(input.groupBy);
+            if (!dateTruncModels) {
+              return [];
+            }
+            const modelLatencies = await getModelLatenciesOverTime(
+              input.projectId,
+              input.filter ?? [],
+              dateTruncModels,
+            );
+
+            return modelLatencies.map((row) => ({
+              model: row.model,
+              startTime: row.start_time,
+              percentile50Duration: row.p50,
+              percentile75Duration: row.p75,
+              percentile90Duration: row.p90,
+              percentile95Duration: row.p95,
+              percentile99Duration: row.p99,
+            })) as DatabaseRow[];
+          case "traces-latencies-aggregated":
+            const traceLatencies = await getTracesLatencies(
+              input.projectId,
+              input.filter ?? [],
+            );
+
+            return traceLatencies.map((row) => ({
+              traceName: row.name,
+              percentile50Duration: row.p50,
+              percentile90Duration: row.p90,
+              percentile95Duration: row.p95,
+              percentile99Duration: row.p99,
+            })) as DatabaseRow[];
+
+          default:
+            throw new TRPCError({
+              code: "NOT_FOUND",
+              message: "Query not found",
+            });
+        }
+      }
       return await executeQuery(ctx.prisma, input.projectId, input);
     }),
   scoreHistogram: protectedProjectProcedure
@@ -34,3 +259,12 @@ export const dashboardRouter = createTRPCRouter({
       return createHistogramData(data);
     }),
 });
+
+const extractTimeSeries = (groupBy?: z.infer<typeof groupByInterface>) => {
+  const temporal = groupBy?.find((group) => {
+    if (group.type === "datetime") {
+      return group;
+    }
+  });
+  return temporal?.type === "datetime" ? temporal.temporalUnit : undefined;
+};

web/src/features/datasets/components/DatasetAggregateTableCell.tsx

@@ -0,0 +1,150 @@
+import { ScoresTableCell } from "@/src/components/scores-table-cell";
+import TableLink from "@/src/components/table/table-link";
+import { Badge } from "@/src/components/ui/badge";
+import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import {
+  type DatasetRunMetric,
+  type RunMetrics,
+} from "@/src/features/datasets/components/DatasetCompareRunsTable";
+import { api } from "@/src/utils/api";
+import { formatIntervalSeconds } from "@/src/utils/dates";
+import { ClockIcon, ListTree } from "lucide-react";
+
+const DatasetAggregateCell = ({
+  scores,
+  resourceMetrics,
+  traceId,
+  projectId,
+  observationId,
+  scoreKeyToDisplayName,
+  selectedMetrics,
+}: RunMetrics & {
+  projectId: string;
+  scoreKeyToDisplayName: Map<string, string>;
+  selectedMetrics: DatasetRunMetric[];
+}) => {
+  // conditionally fetch the trace or observation depending on the presence of observationId
+  const trace = api.traces.byId.useQuery(
+    { traceId, projectId },
+    {
+      enabled: observationId === undefined,
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false,
+      refetchOnWindowFocus: false,
+      refetchOnReconnect: false,
+      staleTime: Infinity,
+    },
+  );
+  const observation = api.observations.byId.useQuery(
+    {
+      observationId: observationId as string, // disabled when observationId is undefined
+      projectId,
+      traceId,
+    },
+    {
+      enabled: observationId !== undefined,
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false,
+      refetchOnWindowFocus: false,
+      refetchOnReconnect: false,
+      staleTime: Infinity,
+    },
+  );
+
+  const data = observationId === undefined ? trace.data : observation.data;
+
+  return (
+    <div className="group flex h-full w-full flex-col gap-1.5 overflow-hidden overflow-y-auto rounded-sm border p-1">
+      <div className="flex flex-row items-center justify-center gap-1">
+        <IOTableCell
+          isLoading={!!!observationId ? trace.isLoading : observation.isLoading}
+          data={data?.output}
+          className={"bg-accent-light-green"}
+          singleLine={true}
+        />
+      </div>
+
+      {selectedMetrics.includes("scores") && (
+        <div className="flex w-full flex-wrap gap-1">
+          {Object.entries(scores).map(([key, score]) => (
+            <Badge
+              variant="outline"
+              className="flex-wrap p-0.5 px-1 font-normal"
+              key={key}
+            >
+              <span className="whitespace-nowrap capitalize">
+                {scoreKeyToDisplayName.get(key)}:
+              </span>
+              <span className="ml-[2px]">
+                <ScoresTableCell aggregate={score} />
+              </span>
+            </Badge>
+          ))}
+        </div>
+      )}
+
+      {selectedMetrics.includes("resourceMetrics") && (
+        <div className="flex w-full flex-row flex-wrap gap-1">
+          <Badge variant="outline" className="p-0.5 px-1 font-normal">
+            <ClockIcon className="mb-0.5 mr-1 h-3 w-3" />
+            <span className="capitalize">
+              {!!resourceMetrics.latency
+                ? formatIntervalSeconds(resourceMetrics.latency)
+                : null}
+            </span>
+          </Badge>
+          <Badge variant="outline" className="p-0.5 px-1 font-normal">
+            <span className="mr-0.5">{resourceMetrics.totalCost}</span>
+          </Badge>
+        </div>
+      )}
+
+      <div className="flex-grow" />
+
+      {observationId ? (
+        <TableLink
+          path={`/project/${projectId}/traces/${encodeURIComponent(traceId)}?observation=${encodeURIComponent(observationId)}`}
+          value={`Trace: ${traceId}, Observation: ${observationId}`}
+          icon={<ListTree className="h-4 w-4" />}
+          className="hidden w-fit self-end group-hover:block"
+        />
+      ) : (
+        <TableLink
+          path={`/project/${projectId}/traces/${encodeURIComponent(traceId)}`}
+          value={`Trace: ${traceId}`}
+          icon={<ListTree className="h-4 w-4" />}
+          className="hidden w-fit self-end group-hover:block"
+        />
+      )}
+    </div>
+  );
+};
+
+export const DatasetAggregateTableCell = ({
+  value,
+  projectId,
+  scoreKeyToDisplayName,
+  selectedMetrics,
+}: {
+  value: RunMetrics;
+  projectId: string;
+  scoreKeyToDisplayName: Map<string, string>;
+  selectedMetrics: DatasetRunMetric[];
+}) => {
+  return value ? (
+    <DatasetAggregateCell
+      projectId={projectId}
+      {...value}
+      scoreKeyToDisplayName={scoreKeyToDisplayName}
+      selectedMetrics={selectedMetrics}
+    />
+  ) : null;
+};

web/src/features/datasets/components/DatasetCompareRunsTable.tsx

@@ -0,0 +1,327 @@
+import { DataTable } from "@/src/components/table/data-table";
+import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
+import TableLink from "@/src/components/table/table-link";
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
+import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
+import { getDatasetRunAggregateColumnProps } from "@/src/features/datasets/components/DatasetRunAggregateColumnHelpers";
+import { useDatasetRunAggregateColumns } from "@/src/features/datasets/hooks/useDatasetRunAggregateColumns";
+import { type ScoreAggregate } from "@langfuse/shared";
+import { type Prisma } from "@langfuse/shared";
+import { NumberParam } from "use-query-params";
+import { useQueryParams, withDefault } from "use-query-params";
+import { useMemo, useState } from "react";
+import { usdFormatter } from "@/src/utils/numbers";
+import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
+import { api, type RouterOutputs } from "@/src/utils/api";
+import { Button } from "@/src/components/ui/button";
+import { ChevronDown, Rows3 } from "lucide-react";
+import {
+  DropdownMenu,
+  DropdownMenuCheckboxItem,
+  DropdownMenuContent,
+  DropdownMenuTrigger,
+} from "@/src/components/ui/dropdown-menu";
+
+export type RunMetrics = {
+  id: string;
+  scores: ScoreAggregate;
+  resourceMetrics: {
+    latency?: number;
+    totalCost?: string;
+  };
+  traceId: string;
+  observationId: string | undefined;
+};
+
+export type RunAggregate = Record<string, RunMetrics>;
+
+export type DatasetCompareRunRowData = {
+  id: string;
+  input: Prisma.JsonValue;
+  expectedOutput: Prisma.JsonValue;
+  metadata: Prisma.JsonValue;
+  // runs holds grouped column with individual run metrics
+  runs?: RunAggregate;
+};
+
+const DATASET_RUN_METRICS = ["scores", "resourceMetrics"] as const;
+export type DatasetRunMetric = (typeof DATASET_RUN_METRICS)[number];
+
+export function DatasetCompareRunsTable(props: {
+  projectId: string;
+  datasetId: string;
+  runIds: string[];
+  runsData?: RouterOutputs["datasets"]["baseRunDataByDatasetId"];
+}) {
+  const [selectedMetrics, setSelectedMetrics] = useState<DatasetRunMetric[]>([
+    "scores",
+    "resourceMetrics",
+  ]);
+  const [isMetricsDropdownOpen, setIsMetricsDropdownOpen] = useState(false);
+  const rowHeight = "l";
+
+  const [paginationState, setPaginationState] = useQueryParams({
+    pageIndex: withDefault(NumberParam, 0),
+    pageSize: withDefault(NumberParam, 50),
+  });
+
+  const baseDatasetItems = api.datasets.baseDatasetItemByDatasetId.useQuery({
+    projectId: props.projectId,
+    datasetId: props.datasetId,
+    page: paginationState.pageIndex,
+    limit: paginationState.pageSize,
+  });
+
+  // Individual queries for each run
+  const runs = (props.runIds ?? []).map((runId) => ({
+    runId,
+    items: api.datasets.runitemsByRunIdOrItemId.useQuery(
+      {
+        projectId: props.projectId,
+        datasetRunId: runId,
+        page: paginationState.pageIndex,
+        limit: paginationState.pageSize,
+      },
+      {
+        refetchOnWindowFocus: false,
+        refetchOnMount: false,
+        refetchOnReconnect: false,
+        staleTime: 5 * 60 * 1000, // 5 minutes
+        enabled: baseDatasetItems.isSuccess,
+      },
+    ),
+  }));
+
+  const combinedData = useMemo(() => {
+    if (!baseDatasetItems.data) return null;
+
+    const runData = runs.reduce<Record<string, RunAggregate>>(
+      (itemsAcc, { runId, items }) => {
+        if (!items.data) return itemsAcc;
+
+        items.data.runItems.forEach(
+          ({ datasetItemId, trace, observation, scores }) => {
+            if (!itemsAcc[datasetItemId]) itemsAcc[datasetItemId] = {};
+
+            itemsAcc[datasetItemId][runId] = {
+              id: runId,
+              traceId: trace?.id ?? "",
+              observationId: observation?.id ?? undefined,
+              resourceMetrics: {
+                latency:
+                  (!!observation ? observation.latency : trace?.duration) ??
+                  undefined,
+                totalCost:
+                  (!!observation?.calculatedTotalCost
+                    ? usdFormatter(observation.calculatedTotalCost.toNumber())
+                    : usdFormatter(trace?.totalCost)) ?? undefined,
+              },
+              scores,
+            };
+          },
+        );
+
+        return itemsAcc;
+      },
+      {},
+    );
+
+    return baseDatasetItems.data?.map(
+      (item): DatasetCompareRunRowData => ({
+        id: item.id,
+        input: item.input ?? "null",
+        expectedOutput: item.expectedOutput ?? "null",
+        metadata: item.metadata ?? "null",
+        runs: runData?.[item.id] || {},
+      }),
+    );
+  }, [baseDatasetItems.data, runs]);
+
+  const scoreKeysAndProps = api.scores.getScoreKeysAndProps.useQuery(
+    {
+      projectId: props.projectId,
+      selectedTimeOption: { filterSource: "TABLE", option: "All time" },
+    },
+    {
+      refetchOnWindowFocus: false,
+      refetchOnMount: false,
+      refetchOnReconnect: false,
+      staleTime: Infinity,
+    },
+  );
+
+  const scoreKeyToDisplayName = useMemo(() => {
+    if (!scoreKeysAndProps.data) return new Map<string, string>();
+    return new Map(
+      scoreKeysAndProps.data.map(({ key, dataType, source, name }) => [
+        key,
+        `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`,
+      ]),
+    );
+  }, [scoreKeysAndProps.data]);
+
+  const { runAggregateColumns, isColumnLoading } =
+    useDatasetRunAggregateColumns({
+      projectId: props.projectId,
+      runIds: props.runIds,
+      runsData: props.runsData ?? [],
+      scoreKeyToDisplayName,
+      cellsLoading: !scoreKeysAndProps.data,
+      selectedMetrics,
+    });
+
+  const columns: LangfuseColumnDef<DatasetCompareRunRowData>[] = [
+    {
+      accessorKey: "id",
+      header: "Item id",
+      id: "id",
+      size: 90,
+      enableHiding: true,
+      defaultHidden: true,
+      cell: ({ row }) => {
+        const id: string = row.getValue("id");
+        return (
+          <TableLink
+            path={`/project/${props.projectId}/datasets/${props.datasetId}/items/${id}`}
+            value={id}
+          />
+        );
+      },
+    },
+    {
+      accessorKey: "input",
+      header: "Input",
+      id: "input",
+      size: 200,
+      enableHiding: true,
+      cell: ({ row }) => {
+        const input = row.getValue(
+          "input",
+        ) as DatasetCompareRunRowData["input"];
+        return input !== null ? <IOTableCell data={input} /> : null;
+      },
+    },
+    {
+      accessorKey: "expectedOutput",
+      header: "Expected Output",
+      id: "expectedOutput",
+      size: 200,
+      enableHiding: true,
+      cell: ({ row }) => {
+        const expectedOutput = row.getValue(
+          "expectedOutput",
+        ) as DatasetCompareRunRowData["expectedOutput"];
+        return expectedOutput !== null ? (
+          <IOTableCell
+            data={expectedOutput}
+            className="bg-accent-light-green"
+          />
+        ) : null;
+      },
+    },
+    {
+      accessorKey: "metadata",
+      header: "Metadata",
+      id: "metadata",
+      size: 200,
+      enableHiding: true,
+      defaultHidden: true,
+      cell: ({ row }) => {
+        const metadata = row.getValue(
+          "metadata",
+        ) as DatasetCompareRunRowData["metadata"];
+        return metadata !== null ? <IOTableCell data={metadata} /> : null;
+      },
+    },
+    {
+      ...getDatasetRunAggregateColumnProps(isColumnLoading),
+      columns: runAggregateColumns,
+    },
+  ];
+
+  const [columnVisibility, setColumnVisibility] =
+    useColumnVisibility<DatasetCompareRunRowData>(
+      "datasetCompareRunsColumnVisibility",
+      columns,
+    );
+
+  return (
+    <>
+      <DataTableToolbar
+        columns={columns}
+        columnVisibility={columnVisibility}
+        setColumnVisibility={setColumnVisibility}
+        rowHeight={rowHeight}
+        actionButtons={
+          <DropdownMenu open={isMetricsDropdownOpen}>
+            <DropdownMenuTrigger asChild>
+              <Button
+                variant="outline"
+                onClick={() => setIsMetricsDropdownOpen(!isMetricsDropdownOpen)}
+              >
+                <Rows3 className="mr-2 h-4 w-4" />
+                <span className="text-xs text-muted-foreground">Metrics</span>
+                <ChevronDown className="ml-2 h-4 w-4" />
+              </Button>
+            </DropdownMenuTrigger>
+            <DropdownMenuContent
+              onPointerDownOutside={() => setIsMetricsDropdownOpen(false)}
+            >
+              <DropdownMenuCheckboxItem
+                checked={selectedMetrics.includes("scores")}
+                onCheckedChange={() => {
+                  setSelectedMetrics((prev) =>
+                    prev.includes("scores")
+                      ? prev.filter((m) => m !== "scores")
+                      : [...prev, "scores"],
+                  );
+                }}
+              >
+                Scores
+              </DropdownMenuCheckboxItem>
+              <DropdownMenuCheckboxItem
+                checked={selectedMetrics.includes("resourceMetrics")}
+                onCheckedChange={() =>
+                  setSelectedMetrics((prev) =>
+                    prev.includes("resourceMetrics")
+                      ? prev.filter((m) => m !== "resourceMetrics")
+                      : [...prev, "resourceMetrics"],
+                  )
+                }
+              >
+                Latency and cost
+              </DropdownMenuCheckboxItem>
+            </DropdownMenuContent>
+          </DropdownMenu>
+        }
+      />
+      <DataTable
+        columns={columns}
+        columnVisibility={columnVisibility}
+        onColumnVisibilityChange={setColumnVisibility}
+        data={
+          baseDatasetItems.isLoading
+            ? { isLoading: true, isError: false }
+            : baseDatasetItems.isError
+              ? {
+                  isLoading: false,
+                  isError: true,
+                  error: baseDatasetItems.error.message,
+                }
+              : {
+                  isLoading: false,
+                  isError: false,
+                  data: combinedData ?? [],
+                }
+        }
+        pagination={{
+          totalCount: baseDatasetItems.data?.length ?? null,
+          onChange: setPaginationState,
+          state: paginationState,
+        }}
+        rowHeight={rowHeight}
+      />
+    </>
+  );
+}

web/src/features/datasets/components/DatasetItemsTable.tsx

@@ -74,7 +74,7 @@ export function DatasetItemsTable({
     if (items.isSuccess) {
       setDetailPageList(
         "datasetItems",
-        items.data.datasetItems.map((t) => t.id),
+        items.data.datasetItems.map((t) => ({ id: t.id })),
       );
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps

web/src/features/datasets/components/DatasetRunAggregateColumnHelpers.tsx

@@ -0,0 +1,73 @@
+import { type LangfuseColumnDef } from "@/src/components/table/types";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { DatasetAggregateTableCell } from "@/src/features/datasets/components/DatasetAggregateTableCell";
+import {
+  type RunMetrics,
+  type RunAggregate,
+  type DatasetCompareRunRowData,
+  type DatasetRunMetric,
+} from "@/src/features/datasets/components/DatasetCompareRunsTable";
+import { type Row } from "@tanstack/react-table";
+import React from "react";
+
+export const constructDatasetRunAggregateColumns = ({
+  runAggregateColumnProps,
+  projectId,
+  scoreKeyToDisplayName,
+  selectedMetrics,
+  cellsLoading = false,
+}: {
+  runAggregateColumnProps: { id: string; name: string; description?: string }[];
+  projectId: string;
+  scoreKeyToDisplayName: Map<string, string>;
+  selectedMetrics: DatasetRunMetric[];
+  cellsLoading?: boolean;
+}): LangfuseColumnDef<DatasetCompareRunRowData>[] => {
+  return runAggregateColumnProps.map((col) => {
+    const { id, name, description } = col;
+
+    return {
+      id,
+      accessorKey: id,
+      header: name,
+      size: 150,
+      ...(description && {
+        headerTooltip: {
+          description,
+        },
+      }),
+      enableHiding: true,
+      cell: ({ row }: { row: Row<DatasetCompareRunRowData> }) => {
+        const runData: RunAggregate = row.getValue("runs") ?? {};
+
+        if (cellsLoading) return <Skeleton className="h-3 w-1/2" />;
+
+        if (!Boolean(Object.keys(runData).length)) return null;
+        if (!runData.hasOwnProperty(id)) return null;
+
+        const value: RunMetrics | undefined = runData[id];
+
+        if (!value) return null;
+        return (
+          <DatasetAggregateTableCell
+            value={value}
+            projectId={projectId}
+            scoreKeyToDisplayName={scoreKeyToDisplayName}
+            selectedMetrics={selectedMetrics}
+          />
+        );
+      },
+    };
+  });
+};
+
+export const getDatasetRunAggregateColumnProps = (isLoading: boolean) => ({
+  accessorKey: "runs",
+  header: "Runs",
+  id: "runs",
+  enableHiding: true,
+  hideByDefault: true,
+  cell: () => {
+    return isLoading ? <Skeleton className="h-3 w-1/2" /> : null;
+  },
+});

web/src/features/datasets/components/DatasetRunItemsTable.tsx

@@ -18,7 +18,7 @@ import {
   getScoreGroupColumnProps,
   verifyAndPrefixScoreDataAgainstKeys,
 } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
-import { type ScoreAggregate } from "@/src/features/scores/lib/types";
+import { type ScoreAggregate } from "@langfuse/shared";
 import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns";
 import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
 
@@ -70,13 +70,15 @@ export function DatasetRunItemsTable(
     if (runItems.isSuccess) {
       setDetailPageList(
         "traces",
-        runItems.data.runItems.filter((i) => !!i.trace).map((i) => i.trace!.id),
+        runItems.data.runItems
+          .filter((i) => !!i.trace)
+          .map((i) => ({ id: i.trace!.id })),
       );
       // set the datasetItems list only when viewing this table from the run page
       if ("datasetRunId" in props)
         setDetailPageList(
           "datasetItems",
-          runItems.data.runItems.map((i) => i.datasetItemId),
+          runItems.data.runItems.map((i) => ({ id: i.datasetItemId })),
         );
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps

web/src/features/datasets/components/DatasetRunsTable.tsx

@@ -7,7 +7,7 @@ import { formatIntervalSeconds } from "@/src/utils/dates";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
 import { type RouterOutput } from "@/src/utils/types";
-import { useEffect } from "react";
+import { useEffect, useState } from "react";
 import { usdFormatter } from "../../../utils/numbers";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
@@ -18,26 +18,26 @@ import {
   getScoreGroupColumnProps,
   verifyAndPrefixScoreDataAgainstKeys,
 } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
-import { type ScoreAggregate } from "@/src/features/scores/lib/types";
+import { type ScoreAggregate } from "@langfuse/shared";
 import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns";
-import { MoreVertical } from "lucide-react";
+import { ChevronDown, Columns3, MoreVertical } from "lucide-react";
 import {
   DropdownMenu,
   DropdownMenuContent,
+  DropdownMenuItem,
   DropdownMenuLabel,
   DropdownMenuTrigger,
 } from "@/src/components/ui/dropdown-menu";
 import { Button } from "@/src/components/ui/button";
 import { DeleteDatasetRunButton } from "@/src/features/datasets/components/DeleteDatasetRunButton";
 import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
-
-type DatasetRunRowKey = {
-  id: string;
-  name: string;
-};
+import { Checkbox } from "@/src/components/ui/checkbox";
+import { type RowSelectionState } from "@tanstack/react-table";
+import Link from "next/link";
 
 export type DatasetRunRowData = {
-  key: DatasetRunRowKey;
+  id: string;
+  name: string;
   createdAt: string;
   countRunItems: string;
   avgLatency: number;
@@ -48,6 +48,46 @@ export type DatasetRunRowData = {
   metadata: Prisma.JsonValue;
 };
 
+const DatasetRunTableMultiSelectAction = ({
+  selectedRunIds,
+  projectId,
+  datasetId,
+}: {
+  selectedRunIds: string[];
+  projectId: string;
+  datasetId: string;
+}) => {
+  return (
+    <>
+      <DropdownMenu>
+        <DropdownMenuTrigger asChild>
+          <Button disabled={selectedRunIds.length < 1}>
+            Actions ({selectedRunIds.length} selected)
+            <ChevronDown className="h-5 w-5" />
+          </Button>
+        </DropdownMenuTrigger>
+        <DropdownMenuContent>
+          <Link
+            href={
+              selectedRunIds.length < 2
+                ? "#"
+                : {
+                    pathname: `/project/${projectId}/datasets/${datasetId}/compare`,
+                    query: { runs: selectedRunIds },
+                  }
+            }
+          >
+            <DropdownMenuItem disabled={selectedRunIds.length < 2}>
+              <Columns3 className="mr-2 h-4 w-4" />
+              <span>Compare</span>
+            </DropdownMenuItem>
+          </Link>
+        </DropdownMenuContent>
+      </DropdownMenu>
+    </>
+  );
+};
+
 export function DatasetRunsTable(props: {
   projectId: string;
   datasetId: string;
@@ -57,6 +97,7 @@ export function DatasetRunsTable(props: {
     pageIndex: withDefault(NumberParam, 0),
     pageSize: withDefault(NumberParam, 50),
   });
+  const [selectedRows, setSelectedRows] = useState<RowSelectionState>({});
 
   const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
     "datasetRuns",
@@ -73,7 +114,7 @@ export function DatasetRunsTable(props: {
     if (runs.isSuccess) {
       setDetailPageList(
         "datasetRuns",
-        runs.data.runs.map((t) => t.id),
+        runs.data.runs.map((t) => ({ id: t.id })),
       );
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -88,17 +129,74 @@ export function DatasetRunsTable(props: {
 
   const columns: LangfuseColumnDef<DatasetRunRowData>[] = [
     {
-      accessorKey: "key",
+      id: "select",
+      accessorKey: "select",
+      size: 30,
+      isPinned: true,
+      header: ({ table }) => {
+        return (
+          <div className="flex h-full items-center">
+            <Checkbox
+              checked={
+                table.getIsAllPageRowsSelected()
+                  ? true
+                  : table.getIsSomePageRowsSelected()
+                    ? "indeterminate"
+                    : false
+              }
+              onCheckedChange={(value) => {
+                table.toggleAllPageRowsSelected(!!value);
+                if (!value) {
+                  setSelectedRows({});
+                }
+              }}
+              aria-label="Select all"
+              className="opacity-60"
+            />
+          </div>
+        );
+      },
+      cell: ({ row }) => {
+        return (
+          <Checkbox
+            checked={row.getIsSelected()}
+            onCheckedChange={(value) => row.toggleSelected(!!value)}
+            aria-label="Select row"
+            className="mt-1 opacity-60 data-[state=checked]:mt-[5px]"
+          />
+        );
+      },
+    },
+    {
+      accessorKey: "id",
+      header: "Id",
+      id: "id",
+      size: 150,
+      enableHiding: true,
+      defaultHidden: true,
+      cell: ({ row }) => {
+        const id: DatasetRunRowData["id"] = row.getValue("id");
+        return (
+          <TableLink
+            path={`/project/${props.projectId}/datasets/${props.datasetId}/runs/${id}`}
+            value={id}
+          />
+        );
+      },
+    },
+    {
+      accessorKey: "name",
       header: "Name",
-      id: "key",
+      id: "name",
       size: 150,
       isPinned: true,
       cell: ({ row }) => {
-        const key: DatasetRunRowData["key"] = row.getValue("key");
+        const name: DatasetRunRowData["name"] = row.getValue("name");
+        const id: DatasetRunRowData["id"] = row.getValue("id");
         return (
           <TableLink
-            path={`/project/${props.projectId}/datasets/${props.datasetId}/runs/${key.id}`}
-            value={key.name}
+            path={`/project/${props.projectId}/datasets/${props.datasetId}/runs/${id}`}
+            value={name}
           />
         );
       },
@@ -169,8 +267,7 @@ export function DatasetRunsTable(props: {
       header: "Actions",
       size: 70,
       cell: ({ row }) => {
-        const key: DatasetRunRowKey = row.getValue("key");
-        const { id: datasetRunId } = key;
+        const id: DatasetRunRowData["id"] = row.getValue("id");
 
         return (
           <DropdownMenu>
@@ -184,7 +281,7 @@ export function DatasetRunsTable(props: {
               <DropdownMenuLabel>Actions</DropdownMenuLabel>
               <DeleteDatasetRunButton
                 projectId={props.projectId}
-                datasetRunId={datasetRunId}
+                datasetRunId={id}
                 fullWidth
               />
             </DropdownMenuContent>
@@ -198,7 +295,8 @@ export function DatasetRunsTable(props: {
     item: RouterOutput["datasets"]["runsByDatasetId"]["runs"][number],
   ): DatasetRunRowData => {
     return {
-      key: { id: item.id, name: item.name },
+      id: item.id,
+      name: item.name,
       createdAt: item.createdAt.toLocaleString(),
       countRunItems: item.countRunItems.toString(),
       avgLatency: item.avgLatency,
@@ -233,7 +331,21 @@ export function DatasetRunsTable(props: {
         setColumnOrder={setColumnOrder}
         rowHeight={rowHeight}
         setRowHeight={setRowHeight}
-        actionButtons={props.menuItems}
+        actionButtons={[
+          props.menuItems,
+          Object.keys(selectedRows).filter((runId) =>
+            runs.data?.runs.map((run) => run.id).includes(runId),
+          ).length > 0 ? (
+            <DatasetRunTableMultiSelectAction
+              // Exclude items that are not in the current page
+              selectedRunIds={Object.keys(selectedRows).filter((runId) =>
+                runs.data?.runs.map((run) => run.id).includes(runId),
+              )}
+              projectId={props.projectId}
+              datasetId={props.datasetId}
+            />
+          ) : null,
+        ]}
       />
       <DataTable
         columns={columns}
@@ -262,6 +374,8 @@ export function DatasetRunsTable(props: {
         columnOrder={columnOrder}
         onColumnOrderChange={setColumnOrder}
         rowHeight={rowHeight}
+        rowSelection={selectedRows}
+        setRowSelection={setSelectedRows}
       />
     </>
   );