chore: release v3.4.0

* fixes: fix user table sorting and filters

* fixes: fix user table sorting and filters

* fixes: fix user table sorting and filters

.env.prod.example

@@ -101,6 +101,7 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_CUSTOM_ISSUER=
 # AUTH_CUSTOM_NAME=
 # AUTH_CUSTOM_SCOPE="openid email profile" # optional
+# AUTH_CUSTOM_CLIENT_AUTH_METHOD="client_secret_basic" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/src/observationsTable.ts

@@ -112,6 +112,14 @@ export const observationsTableCols: ColumnDefinition[] = [
     options: [], // to be added at runtime
     nullable: true,
   },
+  {
+    name: "Model ID",
+    id: "modelId",
+    type: "stringOptions",
+    internal: 'o."internal_model_id"',
+    options: [], // to be added at runtime
+    nullable: true,
+  },
   {
     name: "Input Tokens",
     id: "inputTokens",
@@ -186,6 +194,7 @@ export const observationsTableCols: ColumnDefinition[] = [
 // allows for undefined options, to offer filters while options are still loading
 export type ObservationOptions = {
   model: Array<OptionsDefinition>;
+  modelId: Array<OptionsDefinition>;
   name: Array<OptionsDefinition>;
   traceName: Array<OptionsDefinition>;
   scores_avg: Array<string>;
@@ -194,12 +203,15 @@ export type ObservationOptions = {
 };
 
 export function observationsTableColsWithOptions(
-  options?: ObservationOptions
+  options?: ObservationOptions,
 ): ColumnDefinition[] {
   return observationsTableCols.map((col) => {
     if (col.id === "model") {
       return { ...col, options: options?.model ?? [] };
     }
+    if (col.id === "modelId") {
+      return { ...col, options: options?.modelId ?? [] };
+    }
     if (col.id === "name") {
       return { ...col, options: options?.name ?? [] };
     }

packages/shared/src/server/repositories/observations.ts

@@ -750,6 +750,50 @@ export const getObservationsGroupedByModel = async (
   return res.map((r) => ({ model: r.name }));
 };
 
+export const getObservationsGroupedByModelId = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const observationsFilter = new FilterList([
+    new StringFilter({
+      clickhouseTable: "observations",
+      field: "project_id",
+      operator: "=",
+      value: projectId,
+      tablePrefix: "o",
+    }),
+  ]);
+
+  observationsFilter.push(
+    ...createFilterFromFilterState(
+      filter,
+      observationsTableUiColumnDefinitions,
+    ),
+  );
+
+  const appliedObservationsFilter = observationsFilter.apply();
+
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
+  const query = `
+    SELECT o.internal_model_id as modelId
+    FROM observations o
+    WHERE ${appliedObservationsFilter.query}
+    AND o.type = 'GENERATION'
+    GROUP BY o.internal_model_id
+    ORDER BY count() DESC
+    LIMIT 1000;
+  `;
+
+  const res = await queryClickhouse<{ modelId: string }>({
+    query,
+    params: {
+      ...appliedObservationsFilter.params,
+    },
+  });
+  return res.map((r) => ({ modelId: r.modelId }));
+};
+
 export const getObservationsGroupedByName = async (
   projectId: string,
   filter: FilterState,

packages/shared/src/server/repositories/observations_converters.ts

@@ -126,10 +126,11 @@ export const convertObservation = (
         parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
       : null,
     timeToFirstToken: record.completion_start_time
-      ? parseClickhouseUTCDateTimeFormat(
+      ? (parseClickhouseUTCDateTimeFormat(
           record.completion_start_time,
         ).getTime() -
-        parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
+          parseClickhouseUTCDateTimeFormat(record.start_time).getTime()) /
+        1000
       : null,
   };
 };

packages/shared/src/server/repositories/traces.ts

@@ -680,10 +680,25 @@ export const getTotalUserCount = async (
   });
 };
 
-export const getUserMetrics = async (projectId: string, userIds: string[]) => {
+export const getUserMetrics = async (
+  projectId: string,
+  userIds: string[],
+  filter: FilterState,
+) => {
   if (userIds.length === 0) {
     return [];
   }
+
+  // filter state contains date range filter for traces so far.
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+  );
+  const chFilterRes = chFilter.apply();
+
+  const timestampFilter = chFilter.find(
+    (f) => f.field === "timestamp" && f.operator === ">=",
+  );
+
   // this query uses window functions on observations + traces to always get only the first row and thereby remove deduplicates
   // we filter wherever possible by project id and user id
   const query = `
@@ -713,6 +728,7 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
                     observations o
                 WHERE
                     o.project_id = {projectId: String }
+                    ${timestampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
                     AND o.trace_id in (
                         SELECT
                             distinct id
@@ -721,6 +737,7 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
                         where
                             user_id IN ({userIds: Array(String) })
                             AND project_id = {projectId: String }
+                            ${filter.length > 0 ? `AND ${chFilterRes.query}` : ""}
                     )
                     AND o.type = 'GENERATION'
             ) as o
@@ -740,6 +757,7 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
                 WHERE
                     t.user_id IN ({userIds: Array(String) })
                     AND t.project_id = {projectId: String }
+                    ${filter.length > 0 ? `AND ${chFilterRes.query}` : ""}
             ) as t on t.id = o.trace_id
             and t.project_id = o.project_id
         WHERE
@@ -778,8 +796,17 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
     params: {
       projectId,
       userIds,
+      ...chFilterRes.params,
+      ...(timestampFilter
+        ? {
+            traceTimestamp: convertDateToClickhouseDateTime(
+              (timestampFilter as DateTimeFilter).value,
+            ),
+          }
+        : {}),
     },
   });
+
   return rows.map((row) => ({
     userId: row.user_id,
     maxTimestamp: parseClickhouseUTCDateTimeFormat(row.max_timestamp),

packages/shared/src/server/repositories/traces_converters.ts

@@ -1,7 +1,9 @@
-import { Trace } from "@prisma/client";
+import { Prisma, Trace } from "@prisma/client";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { parseJsonPrioritised } from "../../utils/json";
+import { jsonSchema } from "../../utils/zod";
 
 export const convertTraceDomainToClickhouse = (
   trace: Trace,
@@ -43,8 +45,12 @@ export const convertClickhouseToDomain = (
     userId: record.user_id ?? null,
     sessionId: record.session_id ?? null,
     public: record.public,
-    input: record.input ?? null,
-    output: record.output ?? null,
+    input: (record.input
+      ? jsonSchema.parse(parseJsonPrioritised(record.input))
+      : null) as Prisma.JsonValue | null,
+    output: (record.output
+      ? jsonSchema.parse(parseJsonPrioritised(record.output))
+      : null) as Prisma.JsonValue | null,
     metadata: record.metadata,
     createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
     updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),

packages/shared/src/tableDefinitions/mapObservationsTable.ts

@@ -127,6 +127,12 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "observations",
     clickhouseSelect: 'o."provided_model_name"',
   },
+  {
+    uiTableName: "Model ID",
+    uiTableId: "modelId",
+    clickhouseTableName: "observations",
+    clickhouseSelect: 'o."internal_model_id"',
+  },
   {
     uiTableName: "Input Tokens",
     uiTableId: "inputTokens",

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "private": true,
   "license": "MIT",
   "engines": {

web/src/__tests__/async/dataset-service.servertest.ts

@@ -244,7 +244,7 @@ describe("Fetch datasets for UI presentation", () => {
     expect(JSON.stringify(secondRun.scores)).toEqual(JSON.stringify({}));
   });
 
-  it.only("should test that dataset runs can link to the same traces", async () => {
+  it("should test that dataset runs can link to the same traces", async () => {
     const datasetId = v4();
 
     await prisma.dataset.create({

web/src/__tests__/async/repositories/observation-repository.servertest.ts

@@ -154,8 +154,9 @@ describe("Clickhouse Observations Repository Test", () => {
     expect(firstObservation.promptId).toEqual(observation.prompt_id);
     expect(firstObservation.endTime).toEqual(new Date(observation.end_time));
     expect(firstObservation.timeToFirstToken).toEqual(
-      new Date(observation.completion_start_time).getTime() -
-        new Date(observation.start_time).getTime(),
+      (new Date(observation.completion_start_time).getTime() -
+        new Date(observation.start_time).getTime()) /
+        1000,
     );
     expect(firstObservation.timeToFirstToken).toBeGreaterThan(0);
     expect(firstObservation.calculatedTotalCost).toEqual(

web/src/__tests__/async/repositories/trace-repository.servertest.ts

@@ -35,6 +35,14 @@ describe("Clickhouse Traces Repository Test", () => {
       release: null,
       version: null,
       user_id: null,
+      input: JSON.stringify({
+        this: {
+          is: {
+            a: ["complex", "object"],
+          },
+        },
+      }),
+      output: "regular string",
       created_at: Date.now(),
       updated_at: Date.now(),
       event_ts: Date.now(),
@@ -63,8 +71,8 @@ describe("Clickhouse Traces Repository Test", () => {
     expect(result.userId).toEqual(trace.user_id);
     expect(result.sessionId).toEqual(trace.session_id);
     expect(result.public).toEqual(trace.public);
-    expect(result.input).toEqual(null);
-    expect(result.output).toEqual(null);
+    expect(result.input).toEqual(JSON.parse(trace.input));
+    expect(result.output).toEqual("regular string");
     expect(result.metadata).toEqual(trace.metadata);
     expect(result.createdAt).toEqual(new Date(trace.created_at));
     expect(result.updatedAt).toEqual(new Date(trace.updated_at));

web/src/__tests__/async/users-ui-table.servertest.ts

@@ -52,7 +52,7 @@ describe("getUserMetrics function", () => {
 
     await createObservationsInClickhouse([observation1, observation2]);
 
-    const userMetrics = await getUserMetrics(projectId, [userId]);
+    const userMetrics = await getUserMetrics(projectId, [userId], []);
 
     expect(userMetrics.length).toBe(1);
     expect(userMetrics[0]).toMatchObject({

web/src/components/table/data-table.tsx

@@ -56,6 +56,7 @@ interface DataTableProps<TData, TValue> {
   paginationClassName?: string;
   isBorderless?: boolean;
   shouldRenderGroupHeaders?: boolean;
+  onRowClick?: (row: TData) => void;
 }
 
 export interface AsyncTableData<T> {
@@ -108,6 +109,7 @@ export function DataTable<TData extends object, TValue>({
   paginationClassName,
   isBorderless = false,
   shouldRenderGroupHeaders = false,
+  onRowClick,
 }: DataTableProps<TData, TValue>) {
   const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
   const rowheighttw = getRowHeightTailwindClass(rowHeight);
@@ -319,6 +321,7 @@ export function DataTable<TData extends object, TValue>({
                 columns={columns}
                 data={data}
                 help={help}
+                onRowClick={onRowClick}
               />
             ) : (
               <TableBodyComponent
@@ -327,6 +330,7 @@ export function DataTable<TData extends object, TValue>({
                 columns={columns}
                 data={data}
                 help={help}
+                onRowClick={onRowClick}
               />
             )}
           </Table>
@@ -368,6 +372,7 @@ interface TableBodyComponentProps<TData> {
   columns: LangfuseColumnDef<TData, any>[];
   data: AsyncTableData<TData[]>;
   help?: { description: string; href: string };
+  onRowClick?: (row: TData) => void;
 }
 
 function TableBodyComponent<TData>({
@@ -376,6 +381,7 @@ function TableBodyComponent<TData>({
   columns,
   data,
   help,
+  onRowClick,
 }: TableBodyComponentProps<TData>) {
   return (
     <TableBody>
@@ -390,7 +396,13 @@ function TableBodyComponent<TData>({
         </TableRow>
       ) : table.getRowModel().rows.length ? (
         table.getRowModel().rows.map((row) => (
-          <TableRow key={row.id}>
+          <TableRow
+            key={row.id}
+            onClick={() => onRowClick?.(row.original)}
+            className={
+              onRowClick ? "cursor-pointer hover:bg-accent" : undefined
+            }
+          >
             {row.getVisibleCells().map((cell) => (
               <TableCell
                 key={cell.id}

web/src/components/table/use-cases/generations.tsx

@@ -42,7 +42,8 @@ import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrde
 import { BatchExportTableButton } from "@/src/components/BatchExportTableButton";
 import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 import { BreakdownTooltip } from "@/src/components/trace/BreakdownToolTip";
-import { InfoIcon } from "lucide-react";
+import { InfoIcon, PlusCircle } from "lucide-react";
+import { UpsertModelFormDrawer } from "@/src/features/models/components/UpsertModelFormDrawer";
 
 export type GenerationsTableRow = {
   id: string;
@@ -83,6 +84,7 @@ export type GenerationsTableProps = {
   projectId: string;
   promptName?: string;
   promptVersion?: number;
+  modelId?: string;
   omittedFilter?: string[];
 };
 
@@ -90,6 +92,7 @@ export default function GenerationsTable({
   projectId,
   promptName,
   promptVersion,
+  modelId,
   omittedFilter = [],
 }: GenerationsTableProps) {
   const [searchQuery, setSearchQuery] = useQueryParam(
@@ -143,6 +146,17 @@ export default function GenerationsTable({
       ]
     : [];
 
+  const modelIdFilter: FilterState = modelId
+    ? [
+        {
+          column: "Model ID",
+          type: "string",
+          operator: "=",
+          value: modelId,
+        },
+      ]
+    : [];
+
   const dateRangeFilter: FilterState = dateRange
     ? [
         {
@@ -158,6 +172,7 @@ export default function GenerationsTable({
     ...dateRangeFilter,
     ...promptNameFilter,
     ...promptVersionFilter,
+    ...modelIdFilter,
   ]);
 
   const getCountPayload = {
@@ -447,7 +462,56 @@ export default function GenerationsTable({
       size: 150,
       enableHiding: true,
       enableSorting: true,
+      cell: ({ row }) => {
+        const model = row.getValue("model") as string;
+        const modelId = row.getValue("modelId") as string | undefined;
+
+        if (!model) return null;
+
+        return modelId ? (
+          <TableLink
+            path={`/project/${projectId}/models/${modelId}`}
+            value={model}
+          />
+        ) : (
+          <UpsertModelFormDrawer
+            action="create"
+            projectId={projectId}
+            prefilledModelData={{
+              modelName: model,
+              prices:
+                Object.keys(row.original.usageDetails).length > 0
+                  ? Object.keys(row.original.usageDetails)
+                      .filter((key) => key != "total")
+                      .reduce(
+                        (acc, key) => {
+                          acc[key] = 0.000001;
+                          return acc;
+                        },
+                        {} as Record<string, number>,
+                      )
+                  : undefined,
+            }}
+            className="cursor-pointer"
+          >
+            <span className="flex items-center gap-1">
+              <span>{model}</span>
+              <PlusCircle className="h-3 w-3" />
+            </span>
+          </UpsertModelFormDrawer>
+        );
+      },
     },
+
+    {
+      accessorKey: "modelId",
+      id: "modelId",
+      header: "Model ID",
+      size: 100,
+      enableHiding: true,
+      defaultHidden: true,
+    },
+
     {
       accessorKey: "inputTokens",
       id: "inputTokens",
@@ -691,6 +755,7 @@ export default function GenerationsTable({
             name: generation.name ?? undefined,
             version: generation.version ?? "",
             model: generation.model ?? "",
+            modelId: generation.modelId ?? undefined,
             level: generation.level,
             statusMessage: generation.statusMessage ?? undefined,
             usage: {

web/src/components/table/use-cases/models.tsx

@@ -1,39 +1,40 @@
+import { useEffect } from "react";
+
 import { DataTable } from "@/src/components/table/data-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
-import { Button } from "@/src/components/ui/button";
-import {
-  Popover,
-  PopoverContent,
-  PopoverTrigger,
-} from "@/src/components/ui/popover";
-import { useState } from "react";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
-import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
-import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import { api } from "@/src/utils/api";
-import { usdFormatter } from "@/src/utils/numbers";
-import { type Prisma, type Model } from "@langfuse/shared/src/db";
-import Decimal from "decimal.js";
-import { Trash } from "lucide-react";
+import { type Prisma } from "@langfuse/shared/src/db";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
-import { cn } from "@/src/utils/tailwind";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
+import { type GetModelResult } from "@/src/features/models/validation";
+import { DeleteModelButton } from "@/src/features/models/components/DeleteModelButton";
+import { EditModelButton } from "@/src/features/models/components/EditModelButton";
+import { CloneModelButton } from "@/src/features/models/components/CloneModelButton";
+import { PriceBreakdownTooltip } from "@/src/features/models/components/PriceBreakdownTooltip";
+import { UserCircle2Icon } from "lucide-react";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipTrigger,
+} from "@/src/components/ui/tooltip";
+import { LangfuseIcon } from "@/src/components/LangfuseLogo";
+import { useRouter } from "next/router";
+import { PriceUnitSelector } from "@/src/features/models/components/PriceUnitSelector";
+import { usePriceUnitMultiplier } from "@/src/features/models/hooks/usePriceUnitMultiplier";
 
 export type ModelTableRow = {
   modelId: string;
   maintainer: string;
   modelName: string;
   matchPattern: string;
-  startDate?: Date;
-  inputPrice?: Decimal;
-  outputPrice?: Decimal;
-  totalPrice?: Decimal;
-  unit: string;
+  prices?: Record<string, number>;
   tokenizerId?: string;
   config?: Prisma.JsonValue;
+  serverResponse: GetModelResult;
 };
 
 const modelConfigDescriptions = {
@@ -43,39 +44,46 @@ const modelConfigDescriptions = {
     "Regex pattern to match `model` parameter of generations to model pricing",
   startDate:
     "Date to start pricing model. If not set, model is active unless a more recent version exists.",
-  inputPrice: "Price per 1000 units of input",
-  outputPrice: "Price per 1000 units of output",
-  totalPrice:
-    "Price per 1000 units, for models that don't have input/output specific prices",
-  unit: "Unit of measurement for generative model, can be TOKENS, CHARACTERS, SECONDS, MILLISECONDS, REQUESTS or IMAGES.",
+  prices: "Prices per usage type",
   tokenizerId:
     "Tokenizer used for this model to calculate token counts if none are ingested. Pick from list of supported tokenizers.",
   config:
     "Some tokenizers require additional configuration (e.g. openai tiktoken). See docs for details.",
+  maintainer:
+    "Maintainer of the model. Langfuse managed models can be cloned, user managed models can be edited and deleted. To supersede a Langfuse managed model, set the custom model name to the Langfuse model name.",
 } as const;
 
 export default function ModelTable({ projectId }: { projectId: string }) {
+  const router = useRouter();
   const [paginationState, setPaginationState] = useQueryParams({
     pageIndex: withDefault(NumberParam, 0),
     pageSize: withDefault(NumberParam, 50),
   });
-  const models = api.models.all.useQuery({
-    page: paginationState.pageIndex,
-    limit: paginationState.pageSize,
-    projectId,
-  });
+  const models = api.models.getAll.useQuery(
+    {
+      page: paginationState.pageIndex,
+      limit: paginationState.pageSize,
+      projectId,
+    },
+    {
+      refetchOnWindowFocus: false,
+      refetchOnMount: true,
+      refetchOnReconnect: false,
+      staleTime: 1000 * 60 * 10,
+    },
+  );
   const totalCount = models.data?.totalCount ?? null;
+  const { priceUnit } = usePriceUnitMultiplier();
+  const [rowHeight, setRowHeight] = useRowHeightLocalStorage("models", "m");
 
-  const [rowHeight, setRowHeight] = useRowHeightLocalStorage("models", "s");
+  // Set row height to medium if small as view is not optimized for small row heights
+  useEffect(() => {
+    if (rowHeight === "s") {
+      setRowHeight("m");
+    }
+  }, [rowHeight, setRowHeight]);
 
   const columns: LangfuseColumnDef<ModelTableRow>[] = [
-    {
-      accessorKey: "maintainer",
-      id: "maintainer",
-      enableColumnFilter: true,
-      header: "Maintainer",
-      size: 100,
-    },
     {
       accessorKey: "modelName",
       id: "modelName",
@@ -83,23 +91,40 @@ export default function ModelTable({ projectId }: { projectId: string }) {
       headerTooltip: {
         description: modelConfigDescriptions.modelName,
       },
-      size: 150,
+      cell: ({ row }) => {
+        return (
+          <span className="font-mono text-xs font-semibold">
+            {row.original.modelName}
+          </span>
+        );
+      },
+      size: 120,
     },
     {
-      accessorKey: "startDate",
-      id: "startDate",
-      header: "Start Date",
+      accessorKey: "maintainer",
+      id: "maintainer",
+      header: "Maintainer",
       headerTooltip: {
-        description: modelConfigDescriptions.startDate,
+        description: modelConfigDescriptions.maintainer,
       },
-      size: 100,
+      size: 60,
       cell: ({ row }) => {
-        const value: Date | undefined = row.getValue("startDate");
-
-        return value ? (
-          <span className="text-xs">{value.toISOString().slice(0, 10)} </span>
-        ) : (
-          <span className="text-xs">-</span>
+        const isLangfuse = row.original.maintainer === "Langfuse";
+        return (
+          <div className="flex justify-center">
+            <Tooltip>
+              <TooltipTrigger>
+                {isLangfuse ? (
+                  <LangfuseIcon size={16} />
+                ) : (
+                  <UserCircle2Icon className="h-4 w-4" />
+                )}
+              </TooltipTrigger>
+              <TooltipContent>
+                {isLangfuse ? "Langfuse maintained" : "User maintained"}
+              </TooltipContent>
+            </Tooltip>
+          </div>
         );
       },
     },
@@ -115,104 +140,37 @@ export default function ModelTable({ projectId }: { projectId: string }) {
         const value: string = row.getValue("matchPattern");
 
         return value ? (
-          <IOTableCell data={value} singleLine={rowHeight === "s"} />
+          <span className="font-mono text-xs">{value}</span>
         ) : null;
       },
     },
     {
-      accessorKey: "inputPrice",
-      id: "inputPrice",
+      accessorKey: "prices",
+      id: "prices",
       header: () => {
         return (
-          <>
-            Input Price{" "}
-            <span className="text-xs text-muted-foreground">/ 1k units</span>
-          </>
+          <div className="flex items-center gap-2">
+            <span>Prices {priceUnit}</span>
+            <PriceUnitSelector />
+          </div>
         );
       },
-      headerTooltip: {
-        description: modelConfigDescriptions.inputPrice,
-      },
-      size: 170,
+      size: 120,
       cell: ({ row }) => {
-        const value: Decimal | undefined = row.getValue("inputPrice");
+        const prices: Record<string, number> | undefined =
+          row.getValue("prices");
 
-        return value ? (
-          <span className="text-xs">
-            {usdFormatter(value.toNumber() * 1000, 2, 8)}
-          </span>
-        ) : (
-          <span className="text-xs">-</span>
-        );
-      },
-      enableHiding: true,
-    },
-    {
-      accessorKey: "outputPrice",
-      id: "outputPrice",
-      headerTooltip: {
-        description: modelConfigDescriptions.outputPrice,
-      },
-      header: () => {
         return (
-          <>
-            Output Price{" "}
-            <span className="text-xs text-muted-foreground">/ 1k units</span>
-          </>
-        );
-      },
-      size: 170,
-      cell: ({ row }) => {
-        const value: Decimal | undefined = row.getValue("outputPrice");
-
-        return value ? (
-          <span className="text-xs">
-            {usdFormatter(value.toNumber() * 1000, 2, 8)}
-          </span>
-        ) : (
-          <span className="text-xs">-</span>
+          <PriceBreakdownTooltip
+            modelName={row.original.modelName}
+            prices={prices}
+            priceUnit={priceUnit}
+            rowHeight={rowHeight}
+          />
         );
       },
       enableHiding: true,
     },
-    {
-      accessorKey: "totalPrice",
-      id: "totalPrice",
-      header: () => {
-        return (
-          <>
-            Total Price{" "}
-            <span className="text-xs text-muted-foreground">/ 1k units</span>
-          </>
-        );
-      },
-      headerTooltip: {
-        description: modelConfigDescriptions.totalPrice,
-      },
-      size: 170,
-      cell: ({ row }) => {
-        const value: Decimal | undefined = row.getValue("totalPrice");
-
-        return value ? (
-          <span className="text-xs">
-            {usdFormatter(value.toNumber() * 1000, 2, 8)}
-          </span>
-        ) : (
-          <span className="text-xs">-</span>
-        );
-      },
-      enableHiding: true,
-    },
-    {
-      accessorKey: "unit",
-      id: "unit",
-      header: "Unit",
-      headerTooltip: {
-        description: modelConfigDescriptions.unit,
-      },
-      enableHiding: true,
-      size: 110,
-    },
     {
       accessorKey: "tokenizerId",
       id: "tokenizerId",
@@ -221,7 +179,7 @@ export default function ModelTable({ projectId }: { projectId: string }) {
         description: modelConfigDescriptions.tokenizerId,
       },
       enableHiding: true,
-      size: 110,
+      size: 120,
     },
     {
       accessorKey: "config",
@@ -231,7 +189,7 @@ export default function ModelTable({ projectId }: { projectId: string }) {
         description: modelConfigDescriptions.config,
       },
       enableHiding: true,
-      size: 200,
+      size: 120,
       cell: ({ row }) => {
         const value: Prisma.JsonValue | undefined = row.getValue("config");
 
@@ -243,14 +201,29 @@ export default function ModelTable({ projectId }: { projectId: string }) {
     {
       accessorKey: "actions",
       header: "Actions",
-      size: 70,
+      size: 120,
       cell: ({ row }) => {
-        return (
-          <DeleteModelButton
-            projectId={projectId}
-            modelId={row.original.modelId}
-            isBuiltIn={row.original.maintainer === "Langfuse"}
-          />
+        return row.original.maintainer !== "Langfuse" ? (
+          <div
+            className="flex items-center gap-2"
+            onClick={(e) => e.stopPropagation()}
+          >
+            <EditModelButton
+              projectId={projectId}
+              modelData={row.original.serverResponse}
+            />
+            <DeleteModelButton
+              projectId={projectId}
+              modelData={row.original.serverResponse}
+            />
+          </div>
+        ) : (
+          <div onClick={(e) => e.stopPropagation()}>
+            <CloneModelButton
+              projectId={projectId}
+              modelData={row.original.serverResponse}
+            />
+          </div>
         );
       },
     },
@@ -264,21 +237,16 @@ export default function ModelTable({ projectId }: { projectId: string }) {
     columns,
   );
 
-  const convertToTableRow = (model: Model): ModelTableRow => {
+  const convertToTableRow = (model: GetModelResult): ModelTableRow => {
     return {
       modelId: model.id,
       maintainer: model.projectId ? "User" : "Langfuse",
       modelName: model.modelName,
       matchPattern: model.matchPattern,
-      startDate: model.startDate ? new Date(model.startDate) : undefined,
-      inputPrice: model.inputPrice ? new Decimal(model.inputPrice) : undefined,
-      outputPrice: model.outputPrice
-        ? new Decimal(model.outputPrice)
-        : undefined,
-      totalPrice: model.totalPrice ? new Decimal(model.totalPrice) : undefined,
-      unit: model.unit ?? "",
+      prices: model.prices,
       tokenizerId: model.tokenizerId ?? undefined,
       config: model.tokenizerConfig,
+      serverResponse: model,
     };
   };
 
@@ -320,76 +288,10 @@ export default function ModelTable({ projectId }: { projectId: string }) {
         columnOrder={columnOrder}
         onColumnOrderChange={setColumnOrder}
         rowHeight={rowHeight}
+        onRowClick={(row) => {
+          router.push(`/project/${projectId}/models/${row.modelId}`);
+        }}
       />
     </>
   );
 }
-
-const DeleteModelButton = ({
-  modelId,
-  projectId,
-  isBuiltIn,
-}: {
-  modelId: string;
-  projectId: string;
-  isBuiltIn?: boolean;
-}) => {
-  const [isOpen, setIsOpen] = useState(false);
-  const utils = api.useUtils();
-  const capture = usePostHogClientCapture();
-  const mut = api.models.delete.useMutation({
-    onSuccess: () => {
-      void utils.models.invalidate();
-    },
-  });
-
-  const hasAccess = useHasProjectAccess({
-    projectId,
-    scope: "models:CUD",
-  });
-
-  return (
-    <Popover open={isOpen} onOpenChange={() => setIsOpen(!isOpen)}>
-      <PopoverTrigger asChild>
-        <Button
-          variant="ghost"
-          size="xs"
-          disabled={!hasAccess || isBuiltIn}
-          title={
-            isBuiltIn ? "Built-in models cannot be deleted" : "Delete model"
-          }
-          className={cn(
-            isBuiltIn &&
-              "disabled:pointer-events-auto disabled:cursor-not-allowed",
-          )}
-        >
-          <Trash className="h-4 w-4" />
-        </Button>
-      </PopoverTrigger>
-      <PopoverContent>
-        <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
-        <p className="mb-3 text-sm">
-          This action permanently deletes this model definition.
-        </p>
-        <div className="flex justify-end space-x-4">
-          <Button
-            type="button"
-            variant="destructive"
-            loading={mut.isLoading}
-            onClick={() => {
-              capture("models:delete_button_click");
-              mut.mutateAsync({
-                projectId,
-                modelId,
-              });
-
-              setIsOpen(false);
-            }}
-          >
-            Delete Model
-          </Button>
-        </div>
-      </PopoverContent>
-    </Popover>
-  );
-};

web/src/components/trace/BreakdownToolTip.tsx

@@ -6,6 +6,7 @@ import {
 } from "@/src/components/ui/tooltip";
 import { useState } from "react";
 import Decimal from "decimal.js";
+import { getMaxDecimals } from "@/src/features/models/utils";
 
 interface Details {
   [key: string]: number | undefined;
@@ -36,15 +37,6 @@ export const BreakdownTooltip = ({
       }, {})
     : details;
 
-  // For costs, calculate the maximum number of decimal places needed
-  const getMaxDecimals = (value: number | undefined): number => {
-    if (!value) return 0;
-    const parts = value.toString().split(".");
-
-    // If no decimal point, return 0, else return length of decimal part
-    return parts.length === 1 ? 0 : parts[1].length;
-  };
-
   const formatValueWithPadding = (value: number, maxDecimals: number) => {
     return !value
       ? "0"

web/src/components/trace/ObservationPreview.tsx

@@ -39,7 +39,8 @@ import {
 } from "@/src/components/ui/tabs-bar";
 import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 import { BreakdownTooltip } from "./BreakdownToolTip";
-import { InfoIcon } from "lucide-react";
+import { InfoIcon, PlusCircle } from "lucide-react";
+import { UpsertModelFormDrawer } from "@/src/features/models/components/UpsertModelFormDrawer";
 
 export const ObservationPreview = ({
   observations,
@@ -210,7 +211,47 @@ export const ObservationPreview = ({
                   </Badge>
                 ) : undefined}
                 {preloadedObservation.model ? (
-                  <Badge variant="outline">{preloadedObservation.model}</Badge>
+                  preloadedObservation.modelId ? (
+                    <Badge>
+                      <Link
+                        href={`/project/${preloadedObservation.projectId}/models/${preloadedObservation.modelId}`}
+                        className="flex items-center"
+                        title="View model details"
+                      >
+                        {preloadedObservation.model}
+                      </Link>
+                    </Badge>
+                  ) : (
+                    <UpsertModelFormDrawer
+                      action="create"
+                      projectId={preloadedObservation.projectId}
+                      prefilledModelData={{
+                        modelName: preloadedObservation.model,
+                        prices:
+                          Object.keys(preloadedObservation.usageDetails)
+                            .length > 0
+                            ? Object.keys(preloadedObservation.usageDetails)
+                                .filter((key) => key != "total")
+                                .reduce(
+                                  (acc, key) => {
+                                    acc[key] = 0.000001;
+                                    return acc;
+                                  },
+                                  {} as Record<string, number>,
+                                )
+                            : undefined,
+                      }}
+                      className="cursor-pointer"
+                    >
+                      <Badge
+                        variant="outline"
+                        className="flex items-center gap-1"
+                      >
+                        <span>{preloadedObservation.model}</span>
+                        <PlusCircle className="h-3 w-3" />
+                      </Badge>
+                    </UpsertModelFormDrawer>
+                  )
                 ) : null}
                 {thisCost ? (
                   <BreakdownTooltip

web/src/components/useLocalStorage.tsx

@@ -2,6 +2,7 @@ import { useState, useEffect } from "react";
 
 /**
  * useLocalStorage is a hook for managing data with the localStorage API.
+ * It provides cross-tab synchronization and safe interaction with localStorage.
  *
  * @param {string} localStorageKey - The key under which the value is stored in localStorage.
  * @param {T} initialValue - The initial value of the data to be stored.
@@ -10,49 +11,148 @@ import { useState, useEffect } from "react";
  *
  * @return An array with three elements:
  *     value: Current value
- *     setValue: Function to update the value
+ *     setValue: Function to update the value and sync across tabs
  *     clearValue: Function to remove value from the local storage.
  *                This function will also reset the value to initial value
  *
  * @template T - The type of the data to be stored in localStorage. It should be a type that can be stringified.
  *
  * @throws Will throw an error if the stringifying the value or accessing local storage fails.
+ *
+ * @example
+ * const [theme, setTheme, clearTheme] = useLocalStorage('theme', 'light');
+ * // Use theme value
+ * // Call setTheme to update
+ * // Call clearTheme to reset to 'light'
  */
 function useLocalStorage<T>(
   localStorageKey: string,
   initialValue: T,
 ): [T, React.Dispatch<React.SetStateAction<T>>, () => void] {
+  // Initialize state with value from localStorage or initial value
+  // This initialization is only run once when the component mounts
   const [value, setValue] = useState<T>(() => {
-    if (typeof window === "undefined") {
-      return initialValue;
-    }
+    // Return initial value if running on server-side
+    if (typeof window === "undefined") return initialValue;
+
     try {
-      const storedValue = localStorage.getItem(localStorageKey);
-      return storedValue ? (JSON.parse(storedValue) as T) : initialValue;
+      const stored = localStorage.getItem(localStorageKey);
+      // Parse stored value if it exists, otherwise use initial value
+      return stored ? (JSON.parse(stored) as T) : initialValue;
     } catch (error) {
       console.error("Error reading from local storage", error);
       return initialValue;
     }
   });
 
-  const clearValue = () => {
-    try {
-      localStorage.removeItem(localStorageKey);
-      setValue(initialValue);
-    } catch (error) {
-      console.error("Error clearing local storage", error);
-    }
+  // Helper object to safely interact with localStorage
+  // Handles all error cases and provides consistent interface
+  const safeLocalStorage = {
+    set: (value: T) => {
+      try {
+        const stringified = JSON.stringify(value);
+        localStorage.setItem(localStorageKey, stringified);
+        return stringified;
+      } catch (error) {
+        console.error("Error writing to local storage", error);
+        return null;
+      }
+    },
+    remove: () => {
+      try {
+        localStorage.removeItem(localStorageKey);
+      } catch (error) {
+        console.error("Error clearing local storage", error);
+      }
+    },
   };
 
+  // Function to clear both localStorage and state
+  const clearValue = () => {
+    safeLocalStorage.remove();
+    setValue(initialValue);
+  };
+
+  // Sync to localStorage whenever value changes
+  // This ensures localStorage always has the latest value
   useEffect(() => {
-    try {
-      localStorage.setItem(localStorageKey, JSON.stringify(value));
-    } catch (error) {
-      console.error("Error writing to local storage", error);
-    }
+    safeLocalStorage.set(value);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [localStorageKey, value]);
 
-  return [value, setValue, clearValue] as const;
+  // Handle cross-tab synchronization
+  useEffect(() => {
+    if (typeof window === "undefined") return;
+
+    // Handler for native localStorage events (triggered by other tabs)
+    const handleStorageChange = (e: StorageEvent) => {
+      if (e.key === localStorageKey) {
+        try {
+          setValue(e.newValue ? (JSON.parse(e.newValue) as T) : initialValue);
+        } catch (error) {
+          console.error("Error parsing storage change", error);
+        }
+      }
+    };
+
+    // Handler for custom events (triggered within same tab)
+    const handleCustomEvent = (
+      e: CustomEvent<{ key: string; newValue: string }>,
+    ) => {
+      if (e.detail.key === localStorageKey) {
+        try {
+          setValue(
+            e.detail.newValue
+              ? (JSON.parse(e.detail.newValue) as T)
+              : initialValue,
+          );
+        } catch (error) {
+          console.error("Error parsing custom event", error);
+        }
+      }
+    };
+
+    // Listen for both storage events and custom events
+    window.addEventListener("storage", handleStorageChange);
+    window.addEventListener(
+      "localStorageChange",
+      handleCustomEvent as EventListener,
+    );
+
+    // Cleanup listeners on unmount
+    return () => {
+      window.removeEventListener("storage", handleStorageChange);
+      window.removeEventListener(
+        "localStorageChange",
+        handleCustomEvent as EventListener,
+      );
+    };
+  }, [localStorageKey, initialValue]);
+
+  // Enhanced setValue function that also notifies other tabs
+  const setValueAndNotify: React.Dispatch<React.SetStateAction<T>> = (
+    newValue,
+  ) => {
+    setValue((prev) => {
+      // Handle both direct values and updater functions
+      const resolvedValue =
+        newValue instanceof Function ? newValue(prev) : newValue;
+      const stringified = safeLocalStorage.set(resolvedValue);
+
+      // Dispatch custom event to notify other instances in the same tab
+      if (stringified) {
+        window.dispatchEvent(
+          new CustomEvent("localStorageChange", {
+            detail: { key: localStorageKey, newValue: stringified },
+          }),
+        );
+      }
+
+      return resolvedValue;
+    });
+  };
+
+  return [value, setValueAndNotify, clearValue] as const;
 }
 
 export default useLocalStorage;

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v3.3.0";
+export const VERSION = "v3.4.0";

web/src/ee/features/annotation-queues/components/CreateOrEditAnnotationQueueButton.tsx

@@ -31,7 +31,6 @@ import {
 import { api } from "@/src/utils/api";
 import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
 import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";
-import { CommandItem } from "@/src/components/ui/command";
 import { useRouter } from "next/router";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import {
@@ -39,6 +38,7 @@ import {
   useEntitlementLimit,
 } from "@/src/features/entitlements/hooks";
 import { ActionButton } from "@/src/components/ActionButton";
+import { DropdownMenuItem } from "@/src/components/ui/dropdown-menu";
 
 export const CreateOrEditAnnotationQueueButton = ({
   projectId,
@@ -257,7 +257,7 @@ export const CreateOrEditAnnotationQueueButton = ({
                           };
                         })}
                         controlButtons={
-                          <CommandItem
+                          <DropdownMenuItem
                             onSelect={() => {
                               capture(
                                 "score_configs:manage_configs_item_click",
@@ -269,7 +269,7 @@ export const CreateOrEditAnnotationQueueButton = ({
                             }}
                           >
                             Manage score configs
-                          </CommandItem>
+                          </DropdownMenuItem>
                         }
                       />
                     </FormControl>

web/src/ee/features/experiments/components/CreateExperimentsForm.tsx

@@ -77,6 +77,7 @@ import {
 } from "@/src/components/ui/dialog";
 import Link from "next/link";
 import { useHasEntitlement } from "@/src/features/entitlements/hooks";
+import { DropdownMenuItem } from "@/src/components/ui/dropdown-menu";
 
 const CreateExperimentData = z.object({
   name: z
@@ -790,13 +791,13 @@ export const CreateExperimentsForm = ({
                 }
                 hideClearButton
                 controlButtons={
-                  <CommandItem
+                  <DropdownMenuItem
                     onSelect={() => {
                       window.open(`/project/${projectId}/evals`, "_blank");
                     }}
                   >
                     Manage evaluators
-                  </CommandItem>
+                  </DropdownMenuItem>
                 }
               />
             </FormItem>

web/src/env.mjs

@@ -95,6 +95,17 @@ export const env = createEnv({
     AUTH_CUSTOM_ISSUER: z.string().url().optional(),
     AUTH_CUSTOM_NAME: z.string().optional(),
     AUTH_CUSTOM_SCOPE: z.string().optional(),
+    AUTH_CUSTOM_CLIENT_AUTH_METHOD: z
+      .enum([
+        "client_secret_basic",
+        "client_secret_post",
+        "client_secret_jwt",
+        "private_key_jwt",
+        "tls_client_auth",
+        "self_signed_tls_client_auth",
+        "none",
+      ])
+      .optional(),
     AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT: z.string().optional(),
     AUTH_IGNORE_ACCOUNT_FIELDS: z.string().optional(),
@@ -367,6 +378,8 @@ export const env = createEnv({
     AUTH_CUSTOM_ISSUER: process.env.AUTH_CUSTOM_ISSUER,
     AUTH_CUSTOM_NAME: process.env.AUTH_CUSTOM_NAME,
     AUTH_CUSTOM_SCOPE: process.env.AUTH_CUSTOM_SCOPE,
+    AUTH_CUSTOM_CLIENT_AUTH_METHOD:
+      process.env.AUTH_CUSTOM_CLIENT_AUTH_METHOD,
     AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING,
     AUTH_IGNORE_ACCOUNT_FIELDS:

web/src/features/dashboard/components/score-analytics/CategoricalScoreChart.tsx

@@ -1,31 +1,19 @@
 import { api } from "@/src/utils/api";
-
-import {
-  type ScoreSource,
-  type FilterState,
-  type ScoreDataType,
-} from "@langfuse/shared";
+import { type FilterState } from "@langfuse/shared";
 import { createTracesTimeFilter } from "@/src/features/dashboard/lib/dashboard-utils";
 import {
   type DashboardDateRangeAggregationOption,
   dashboardDateRangeAggregationSettings,
 } from "@/src/utils/date-range-utils";
 import React, { useMemo } from "react";
-import { BarChart } from "@tremor/react";
-import { Card } from "@/src/components/ui/card";
-import { getColorsForCategories } from "@/src/features/dashboard/utils/getColorsForCategories";
-import {
-  isEmptyBarChart,
-  transformCategoricalScoresToChartData,
-} from "@/src/features/dashboard/lib/score-analytics-utils";
-import { NoDataOrLoading } from "@/src/components/NoDataOrLoading";
 import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
+import { DashboardCategoricalScoreAdapter } from "@/src/features/scores/adapters";
+import { type ScoreData } from "@/src/features/scores/types";
+import { CategoricalChart } from "@/src/features/scores/components/ScoreChart";
 
 export function CategoricalScoreChart(props: {
   projectId: string;
-  name: string;
-  source: ScoreSource;
-  dataType: ScoreDataType;
+  scoreData: ScoreData;
   globalFilterState: FilterState;
   agg?: DashboardDateRangeAggregationOption;
 }) {
@@ -45,19 +33,19 @@ export function CategoricalScoreChart(props: {
         {
           type: "string",
           column: "scoreName",
-          value: props.name,
+          value: props.scoreData.name,
           operator: "=",
         },
         {
           type: "string",
           column: "scoreSource",
-          value: props.source,
+          value: props.scoreData.source,
           operator: "=",
         },
         {
           type: "string",
           column: "scoreDataType",
-          value: props.dataType,
+          value: props.scoreData.dataType,
           operator: "=",
         },
       ],
@@ -99,40 +87,23 @@ export function CategoricalScoreChart(props: {
   );
 
   const { chartData, chartLabels } = useMemo(() => {
-    return scores.data
-      ? transformCategoricalScoresToChartData(
-          scores.data,
-          "scoreTimestamp",
-          props.agg,
-        )
-      : { chartData: [], chartLabels: [] };
+    if (!scores.data) return { chartData: [], chartLabels: [] };
+
+    const adapter = new DashboardCategoricalScoreAdapter(
+      scores.data,
+      "scoreTimestamp",
+      props.agg,
+    );
+    return adapter.toChartData();
   }, [scores.data, props.agg]);
 
-  const barCategoryGap = (chartLength: number): string => {
-    if (chartLength > 7) return "10%";
-    if (chartLength > 5) return "20%";
-    if (chartLength > 3) return "30%";
-    else return "40%";
-  };
-  const colors = getColorsForCategories(chartLabels);
-
-  return isEmptyBarChart({ data: chartData }) ? (
-    <NoDataOrLoading isLoading={scores.isLoading} />
-  ) : (
-    <Card className="min-h-[9rem] w-full flex-1 rounded-tremor-default border">
-      <BarChart
-        className="mt-4"
-        data={chartData}
-        index="binLabel"
-        categories={chartLabels}
-        colors={colors}
-        valueFormatter={(number: number) =>
-          Intl.NumberFormat("en-US").format(number).toString()
-        }
-        yAxisWidth={48}
-        barCategoryGap={barCategoryGap(chartData.length)}
-        stack={!!props.agg}
-      />
-    </Card>
+  return (
+    <CategoricalChart
+      chartData={chartData}
+      chartLabels={chartLabels}
+      isLoading={scores.isLoading}
+      className="min-h-[9rem] flex-1"
+      stack={!!props.agg}
+    />
   );
 }

web/src/features/dashboard/components/score-analytics/ScoreAnalytics.tsx

@@ -127,19 +127,17 @@ export function ScoreAnalytics(props: {
                     {(isCategoricalDataType(dataType) ||
                       isBooleanDataType(dataType)) && (
                       <CategoricalScoreChart
-                        source={source}
-                        name={name}
-                        dataType={dataType}
                         projectId={props.projectId}
+                        scoreData={scoreData}
                         globalFilterState={props.globalFilterState}
                       />
                     )}
                     {isNumericDataType(dataType) && (
                       <NumericScoreHistogram
+                        projectId={props.projectId}
                         source={source}
                         name={name}
                         dataType={dataType}
-                        projectId={props.projectId}
                         globalFilterState={props.globalFilterState}
                       />
                     )}
@@ -154,11 +152,9 @@ export function ScoreAnalytics(props: {
                     {(isCategoricalDataType(dataType) ||
                       isBooleanDataType(dataType)) && (
                       <CategoricalScoreChart
-                        agg={props.agg}
-                        source={source}
-                        name={name}
-                        dataType={dataType}
                         projectId={props.projectId}
+                        agg={props.agg}
+                        scoreData={scoreData}
                         globalFilterState={props.globalFilterState}
                       />
                     )}

web/src/features/dashboard/lib/score-analytics-utils.ts

@@ -1,16 +1,36 @@
 import { type DashboardDateRangeAggregationOption } from "@/src/utils/date-range-utils";
 import { type DatabaseRow } from "@/src/server/api/services/queryBuilder";
+import {
+  type CategoryCounts,
+  type ChartBin,
+  type HistogramBin,
+} from "@/src/features/scores/types";
+import { type RouterOutputs } from "@/src/utils/api";
 
-// types
-type HistogramBin = { binLabel: string; count: number };
-type CategoryCounts = Record<string, number>;
-type ChartBin = { binLabel: string } & CategoryCounts;
+export const RESOURCE_METRICS = [
+  {
+    key: "latency",
+    value: "Latency",
+    objectKey: "avgLatency",
+    label: "Latency (ms)",
+  },
+  {
+    key: "cost",
+    value: "Cost",
+    objectKey: "avgTotalCost",
+    label: "Average Total Cost ($)",
+  },
+];
 
 // numeric score analytics helpers
 function round(value: number, precision = 2) {
   return parseFloat(value.toFixed(precision));
 }
 
+export function uniqueAndSort(labels: string[]): string[] {
+  return Array.from(new Set(labels)).sort();
+}
+
 function computeBinSize(
   minBins: number,
   maxBins: number,
@@ -125,8 +145,112 @@ function groupCategoricalScoreDataByTimestamp(
   );
 }
 
-function uniqueAndSort(labels: string[]): string[] {
-  return Array.from(new Set(labels)).sort();
+type ChartAccumulator = Map<
+  string,
+  { chartData: ChartBin[]; chartLabels: string[] }
+>;
+
+function initializeOrGetChartData(acc: ChartAccumulator, key: string) {
+  if (!acc.has(key)) {
+    acc.set(key, { chartData: [], chartLabels: [] });
+  }
+  return acc.get(key)!;
+}
+
+function createNumericScoreData(run: string, score: number, scoreName: string) {
+  return {
+    chartLabels: [scoreName],
+    chartBin: {
+      binLabel: run,
+      [scoreName]: score,
+    } as ChartBin,
+  };
+}
+
+function createCategoricalScoreData(
+  run: string,
+  valueCounts: Array<{ value: string; count: number }>,
+  values: string[],
+) {
+  const categoryCounts = valueCounts.reduce(
+    (counts, { value, count }) => ({
+      ...counts,
+      [value]: count,
+    }),
+    {} as CategoryCounts,
+  );
+
+  return {
+    chartLabels: values,
+    chartBin: {
+      binLabel: run,
+      ...categoryCounts,
+    } as ChartBin,
+  };
+}
+
+function addMetricToAccumulator(
+  acc: ChartAccumulator,
+  key: string,
+  chartBin: ChartBin,
+  chartLabels: string[],
+) {
+  const current = initializeOrGetChartData(acc, key);
+  acc.set(key, {
+    chartData: [...current.chartData, chartBin],
+    chartLabels,
+  });
+}
+
+export function transformAggregatedRunMetricsToChartData(
+  runMetrics: RouterOutputs["datasets"]["runsByDatasetIdMetrics"]["runs"],
+  scoreIdToName: Map<string, string>,
+) {
+  const reversedMetrics = runMetrics.slice().reverse();
+
+  return reversedMetrics.reduce((acc, run) => {
+    // Handle scores
+    Object.entries(run.scores ?? {}).forEach(([scoreId, score]) => {
+      const scoreData =
+        score.type === "NUMERIC"
+          ? createNumericScoreData(
+              run.name,
+              score.average,
+              scoreIdToName.get(scoreId) ?? scoreId,
+            )
+          : createCategoricalScoreData(
+              run.name,
+              score.valueCounts,
+              score.values,
+            );
+
+      addMetricToAccumulator(
+        acc,
+        scoreId,
+        scoreData.chartBin,
+        scoreData.chartLabels,
+      );
+    });
+
+    // Handle resource metrics
+    RESOURCE_METRICS.forEach(({ key, objectKey }) => {
+      const resourceValue = run[objectKey as keyof typeof run];
+      const resourceData = createNumericScoreData(
+        run.name,
+        !!resourceValue ? Number(resourceValue) : 0,
+        key,
+      );
+
+      addMetricToAccumulator(
+        acc,
+        key,
+        resourceData.chartBin,
+        resourceData.chartLabels,
+      );
+    });
+
+    return acc;
+  }, new Map());
 }
 
 export function transformCategoricalScoresToChartData(
@@ -155,11 +279,11 @@ export function transformCategoricalScoresToChartData(
       chartData.push({ ...categoryCounts, binLabel: timestamp } as ChartBin);
     });
 
-    return { chartData, chartLabels: uniqueAndSort(chartLabels) };
+    return { chartData, chartLabels };
   }
 }
 
-export function isEmptyBarChart({ data }: { data: ChartBin[] }) {
+export function isEmptyChart({ data }: { data: ChartBin[] }) {
   return (
     data.length === 0 || data.every((item) => Object.keys(item).length === 1)
   );

web/src/features/datasets/components/DatasetAnalytics.tsx

@@ -0,0 +1,36 @@
+import { RESOURCE_METRICS } from "@/src/features/dashboard/lib/score-analytics-utils";
+import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";
+import { ChartColumnBig } from "lucide-react";
+
+export function DatasetAnalytics(props: {
+  projectId: string;
+  scoreOptions: { key: string; value: string }[];
+  selectedMetrics: string[];
+  setSelectedMetrics: (metrics: string[]) => void;
+}) {
+  return (
+    <MultiSelectKeyValues
+      className="max-w-fit"
+      placeholder="Search..."
+      title="Charts"
+      iconLeft={<ChartColumnBig className="mr-1 h-4 w-4" />}
+      hideClearButton
+      onValueChange={(values, changedValue, selectedKeys) => {
+        if (values.length === 0) props.setSelectedMetrics([]);
+
+        if (changedValue) {
+          if (selectedKeys?.has(changedValue)) {
+            props.setSelectedMetrics([...props.selectedMetrics, changedValue]);
+          } else {
+            props.setSelectedMetrics(
+              props.selectedMetrics.filter((key) => key !== changedValue),
+            );
+          }
+        }
+      }}
+      values={props.selectedMetrics}
+      options={RESOURCE_METRICS}
+      groupedOptions={[{ label: "Scores", options: props.scoreOptions }]}
+    />
+  );
+}

web/src/features/datasets/components/DatasetRunsTable.tsx

@@ -6,7 +6,7 @@ import { api } from "@/src/utils/api";
 import { formatIntervalSeconds } from "@/src/utils/dates";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 import { type RouterOutput } from "@/src/utils/types";
-import { useEffect, useState } from "react";
+import { useEffect, useMemo, useState } from "react";
 import { usdFormatter } from "../../../utils/numbers";
 import { DataTableToolbar } from "@/src/components/table/data-table-toolbar";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
@@ -14,6 +14,7 @@ import { type Prisma } from "@langfuse/shared";
 import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
 import { IOTableCell } from "@/src/components/ui/CodeJsonViewer";
 import {
+  getScoreDataTypeIcon,
   getScoreGroupColumnProps,
   verifyAndPrefixScoreDataAgainstKeys,
 } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
@@ -36,6 +37,14 @@ import Link from "next/link";
 import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
 import { joinTableCoreAndMetrics } from "@/src/components/table/utils/joinTableCoreAndMetrics";
 import { Skeleton } from "@/src/components/ui/skeleton";
+import {
+  RESOURCE_METRICS,
+  transformAggregatedRunMetricsToChartData,
+} from "@/src/features/dashboard/lib/score-analytics-utils";
+import { TimeseriesChart } from "@/src/features/scores/components/TimeseriesChart";
+import { Card, CardContent } from "@/src/components/ui/card";
+import { CompareViewAdapter } from "@/src/features/scores/adapters";
+import { isNumericDataType } from "@/src/features/scores/lib/helpers";
 
 export type DatasetRunRowData = {
   id: string;
@@ -93,6 +102,8 @@ const DatasetRunTableMultiSelectAction = ({
 export function DatasetRunsTable(props: {
   projectId: string;
   datasetId: string;
+  selectedMetrics: string[];
+  setScoreOptions: (options: { key: string; value: string }[]) => void;
   menuItems?: React.ReactNode;
 }) {
   const [paginationState, setPaginationState] = useQueryParams({
@@ -105,6 +116,8 @@ export function DatasetRunsTable(props: {
     "datasetRuns",
     "s",
   );
+  const { setScoreOptions } = props;
+
   const runs = api.datasets.runsByDatasetId.useQuery({
     projectId: props.projectId,
     datasetId: props.datasetId,
@@ -149,6 +162,37 @@ export function DatasetRunsTable(props: {
       showAggregateViewOnly: true,
     });
 
+  const scoreIdToName = useMemo(() => {
+    return new Map(scoreKeysAndProps.map((obj) => [obj.key, obj.name]) ?? []);
+  }, [scoreKeysAndProps]);
+
+  const runAggregatedMetrics = useMemo(() => {
+    return transformAggregatedRunMetricsToChartData(
+      runsMetrics.data?.runs ?? [],
+      scoreIdToName,
+    );
+  }, [runsMetrics.data, scoreIdToName]);
+
+  const { scoreAnalyticsOptions, scoreKeyToData } = useMemo(() => {
+    const scoreAnalyticsOptions = scoreKeysAndProps
+      ? scoreKeysAndProps.map(({ key, name, dataType, source }) => ({
+          key,
+          value: `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`,
+        }))
+      : [];
+
+    return {
+      scoreAnalyticsOptions,
+      scoreKeyToData: new Map(
+        scoreKeysAndProps.map((obj) => [obj.key, obj]) ?? [],
+      ),
+    };
+  }, [scoreKeysAndProps]);
+
+  useEffect(() => {
+    setScoreOptions(scoreAnalyticsOptions);
+  }, [scoreAnalyticsOptions, setScoreOptions]);
+
   const columns: LangfuseColumnDef<DatasetRunRowData>[] = [
     {
       id: "select",
@@ -348,6 +392,51 @@ export function DatasetRunsTable(props: {
 
   return (
     <>
+      {Boolean(props.selectedMetrics.length) &&
+        Boolean(runAggregatedMetrics?.size) && (
+          <Card className="my-4 max-h-[25dvh] md:max-h-[30dvh]">
+            <CardContent className="mt-2 h-full">
+              <div className="flex h-full w-full gap-4 overflow-x-auto">
+                {props.selectedMetrics.map((key) => {
+                  const adapter = new CompareViewAdapter(
+                    runAggregatedMetrics,
+                    key,
+                  );
+                  const { chartData, chartLabels } = adapter.toChartData();
+
+                  const scoreData = scoreKeyToData.get(key);
+                  if (!scoreData)
+                    return (
+                      <TimeseriesChart
+                        key={key}
+                        chartData={chartData}
+                        chartLabels={chartLabels}
+                        title={
+                          RESOURCE_METRICS.find((metric) => metric.key === key)
+                            ?.label ?? key
+                        }
+                        type="numeric"
+                      />
+                    );
+
+                  return (
+                    <TimeseriesChart
+                      key={key}
+                      chartData={chartData}
+                      chartLabels={chartLabels}
+                      title={`${getScoreDataTypeIcon(scoreData.dataType)} ${scoreData.name} (${scoreData.source.toLowerCase()})`}
+                      type={
+                        isNumericDataType(scoreData.dataType)
+                          ? "numeric"
+                          : "categorical"
+                      }
+                    />
+                  );
+                })}
+              </div>
+            </CardContent>
+          </Card>
+        )}
       <DataTableToolbar
         columns={columns}
         columnVisibility={columnVisibility}

web/src/features/datasets/server/dataset-router.ts

@@ -858,8 +858,9 @@ async function runsByDatasetIdPg(
     projectId: string;
     datasetId: string;
     queryClickhouse: boolean;
-    page: number;
-    limit: number;
+    page?: number;
+    limit?: number;
+    runIds?: string[];
   },
 ) {
   const scoresByRunId = await prisma.$queryRaw<
@@ -889,10 +890,11 @@ async function runsByDatasetIdPg(
           runs.dataset_id = ${input.datasetId}
           AND runs.project_id = ${input.projectId}
           AND s.score IS NOT NULL
+          ${input.runIds ? Prisma.sql`AND runs.id IN (${Prisma.join(input.runIds)})` : Prisma.empty}  
         GROUP BY
           runs.id
-        LIMIT ${input.limit}
-        OFFSET ${input.page * input.limit}
+        ${input.limit ? Prisma.sql`LIMIT ${input.limit}` : Prisma.empty}
+        ${input.page && input.limit ? Prisma.sql`OFFSET ${input.page * input.limit}` : Prisma.empty}
       `);
 
   const runs = await prisma.$queryRaw<
@@ -983,10 +985,11 @@ async function runsByDatasetIdPg(
         WHERE
           runs.dataset_id = ${input.datasetId}
           AND runs.project_id = ${input.projectId}
+          ${input.runIds ? Prisma.sql`AND runs.id IN (${Prisma.join(input.runIds)})` : Prisma.empty}
         ORDER BY
           runs.created_at DESC
-        LIMIT ${input.limit}
-        OFFSET ${input.page * input.limit}
+        ${input.limit ? Prisma.sql`LIMIT ${input.limit}` : Prisma.empty}
+        ${input.page && input.limit ? Prisma.sql`OFFSET ${input.page * input.limit}` : Prisma.empty}
       `);
 
   const totalRuns = await prisma.datasetRuns.count({
@@ -1002,7 +1005,7 @@ async function runsByDatasetIdPg(
       ...run,
       scores: aggregateScores(
         scoresByRunId.flatMap((s) => (s.runId === run.id ? s.scores : [])),
-      ) as ScoreAggregate | undefined
+      ) as ScoreAggregate | undefined,
     })),
   };
 }

web/src/features/datasets/server/service.ts

@@ -1,9 +1,9 @@
 import {
   filterAndValidateDbScoreList,
-  paginationZod,
   Prisma,
   type PrismaClient,
   type DatasetRunItems,
+  optionalPaginationZod,
 } from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
 import { v4 } from "uuid";
@@ -32,7 +32,8 @@ export const datasetRunsTableSchema = z.object({
   projectId: z.string(),
   datasetId: z.string(),
   queryClickhouse: z.boolean().optional().default(false),
-  ...paginationZod,
+  runIds: z.array(z.string()).optional(),
+  ...optionalPaginationZod,
 });
 
 type PostgresRunItem = {
@@ -235,10 +236,11 @@ export const getDatasetRunsFromPostgres = async (
       WHERE
         d.id = ${input.datasetId}
         AND d.project_id = ${input.projectId}
+        ${input.runIds?.length ? Prisma.sql`AND runs.id IN (${Prisma.join(input.runIds)})` : Prisma.empty}
       GROUP BY runs.id, runs.name, runs.description, runs.metadata, runs.created_at, runs.updated_at
       ORDER BY runs.created_at DESC
-      LIMIT ${input.limit}
-      OFFSET ${input.page * input.limit}
+      ${input.limit ? Prisma.sql`LIMIT ${input.limit}` : Prisma.empty}
+      ${input.page && input.limit ? Prisma.sql`OFFSET ${input.page * input.limit}` : Prisma.empty}
     `,
   );
 };

web/src/features/models/components/CloneModelButton.tsx

@@ -0,0 +1,30 @@
+import { Button } from "@/src/components/ui/button";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { UpsertModelFormDrawer } from "@/src/features/models/components/UpsertModelFormDrawer";
+import { type GetModelResult } from "@/src/features/models/validation";
+
+export const CloneModelButton = ({
+  modelData,
+  projectId,
+}: {
+  modelData: GetModelResult;
+  projectId: string;
+}) => {
+  const hasAccess = useHasProjectAccess({
+    projectId,
+    scope: "models:CUD",
+  });
+
+  return (
+    <UpsertModelFormDrawer {...{ modelData, projectId, action: "clone" }}>
+      <Button
+        variant="outline"
+        disabled={!hasAccess}
+        title="Clone model"
+        className="flex items-center"
+      >
+        <span>Clone</span>
+      </Button>
+    </UpsertModelFormDrawer>
+  );
+};

web/src/features/models/components/DeleteModelButton.tsx

@@ -0,0 +1,76 @@
+import { useState } from "react";
+
+import { Button } from "@/src/components/ui/button";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import { type GetModelResult } from "@/src/features/models/validation";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { api } from "@/src/utils/api";
+
+export const DeleteModelButton = ({
+  modelData,
+  projectId,
+  onSuccess,
+}: {
+  modelData: GetModelResult;
+  projectId: string;
+  onSuccess?: () => void;
+}) => {
+  const [isOpen, setIsOpen] = useState(false);
+  const utils = api.useUtils();
+  const capture = usePostHogClientCapture();
+  const mut = api.models.delete.useMutation({
+    onSuccess: () => {
+      void utils.models.invalidate();
+      onSuccess?.();
+    },
+  });
+
+  const hasAccess = useHasProjectAccess({
+    projectId,
+    scope: "models:CUD",
+  });
+
+  return (
+    <Popover open={isOpen} onOpenChange={() => setIsOpen(!isOpen)}>
+      <PopoverTrigger asChild>
+        <Button
+          variant="outline"
+          title="Delete model"
+          disabled={!hasAccess}
+          className="flex items-center border-light-red"
+        >
+          <span className="text-dark-red">Delete</span>
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent>
+        <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
+        <p className="mb-3 text-sm">
+          This action permanently deletes this model definition.
+        </p>
+        <div className="flex justify-end space-x-4">
+          <Button
+            type="button"
+            variant="destructive"
+            loading={mut.isLoading}
+            onClick={() => {
+              capture("models:delete_button_click");
+              mut.mutateAsync({
+                projectId,
+                modelId: modelData.id,
+              });
+
+              setIsOpen(false);
+            }}
+          >
+            Delete Model
+          </Button>
+        </div>
+      </PopoverContent>
+    </Popover>
+  );
+};

web/src/features/models/components/EditModelButton.tsx

@@ -0,0 +1,30 @@
+import { Button } from "@/src/components/ui/button";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { UpsertModelFormDrawer } from "@/src/features/models/components/UpsertModelFormDrawer";
+import { type GetModelResult } from "@/src/features/models/validation";
+
+export const EditModelButton = ({
+  modelData,
+  projectId,
+}: {
+  modelData: GetModelResult;
+  projectId: string;
+}) => {
+  const hasAccess = useHasProjectAccess({
+    projectId,
+    scope: "models:CUD",
+  });
+
+  return (
+    <UpsertModelFormDrawer {...{ modelData, projectId, action: "edit" }}>
+      <Button
+        variant="outline"
+        disabled={!hasAccess}
+        title="Edit model"
+        className="flex items-center"
+      >
+        <span>Edit</span>
+      </Button>
+    </UpsertModelFormDrawer>
+  );
+};

web/src/features/models/components/NewModelForm.tsx

@@ -1,456 +0,0 @@
-import { useState } from "react";
-import { useForm } from "react-hook-form";
-import * as z from "zod";
-
-import { DatePicker } from "@/src/components/date-picker";
-import Header from "@/src/components/layouts/header";
-import { Button } from "@/src/components/ui/button";
-import {
-  Form,
-  FormControl,
-  FormDescription,
-  FormField,
-  FormItem,
-  FormLabel,
-  FormMessage,
-} from "@/src/components/ui/form";
-import { Input } from "@/src/components/ui/input";
-import {
-  Select,
-  SelectContent,
-  SelectItem,
-  SelectTrigger,
-  SelectValue,
-} from "@/src/components/ui/select";
-import { ModelUsageUnit } from "@langfuse/shared";
-import { AutoComplete } from "@/src/features/prompts/components/auto-complete";
-import { api } from "@/src/utils/api";
-import { zodResolver } from "@hookform/resolvers/zod";
-import { JsonEditor } from "@/src/components/json-editor";
-import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
-import Link from "next/link";
-import { utcDate } from "@/src/utils/dates";
-
-const formSchema = z.object({
-  modelName: z.string().min(1),
-  matchPattern: z.string(),
-  startDate: z.date().optional(),
-  inputPrice: z
-    .string()
-    .refine((value) => value === "" || isFinite(parseFloat(value)), {
-      message: "Price needs to be numeric",
-    })
-    .optional(),
-  outputPrice: z
-    .string()
-    .refine((value) => value === "" || isFinite(parseFloat(value)), {
-      message: "Price needs to be numeric",
-    })
-    .optional(),
-  totalPrice: z
-    .string()
-    .refine((value) => value === "" || isFinite(parseFloat(value)), {
-      message: "Price needs to be numeric",
-    })
-    .optional(),
-  unit: z.nativeEnum(ModelUsageUnit),
-  tokenizerId: z.enum(["openai", "claude", "None"]),
-  tokenizerConfig: z.string().refine(
-    (value) => {
-      try {
-        JSON.parse(value);
-        return true;
-      } catch (e) {
-        return false;
-      }
-    },
-    {
-      message: "Tokenizer config needs to be valid JSON",
-    },
-  ),
-});
-
-export const NewModelForm = (props: {
-  projectId: string;
-  onFormSuccess?: () => void;
-}) => {
-  const [formError, setFormError] = useState<string | null>(null);
-  const capture = usePostHogClientCapture();
-  const form = useForm<z.infer<typeof formSchema>>({
-    resolver: zodResolver(formSchema),
-    defaultValues: {
-      modelName: "",
-      matchPattern: "",
-      startDate: undefined,
-      inputPrice: "",
-      outputPrice: "",
-      totalPrice: "",
-      unit: ModelUsageUnit.Tokens,
-      tokenizerId: "None",
-      tokenizerConfig: "{}",
-    },
-  });
-
-  const utils = api.useUtils();
-  const createModelMutation = api.models.create.useMutation({
-    onSuccess: () => utils.models.invalidate(),
-    onError: (error) => setFormError(error.message),
-  });
-
-  const modelNames = api.models.modelNames.useQuery({
-    projectId: props.projectId,
-  });
-
-  function onSubmit(values: z.infer<typeof formSchema>) {
-    capture("models:new_form_submit");
-    createModelMutation
-      .mutateAsync({
-        projectId: props.projectId,
-        modelName: values.modelName,
-        matchPattern: values.matchPattern,
-        inputPrice: !!values.inputPrice
-          ? parseFloat(values.inputPrice)
-          : undefined,
-        outputPrice: !!values.outputPrice
-          ? parseFloat(values.outputPrice)
-          : undefined,
-        totalPrice: !!values.totalPrice
-          ? parseFloat(values.totalPrice)
-          : undefined,
-        unit: values.unit,
-        tokenizerId:
-          values.tokenizerId === "None" ? undefined : values.tokenizerId,
-        tokenizerConfig:
-          values.tokenizerConfig &&
-          typeof JSON.parse(values.tokenizerConfig) === "object"
-            ? (JSON.parse(values.tokenizerConfig) as Record<string, number>)
-            : undefined,
-        startDate: values.startDate ? utcDate(values.startDate) : undefined,
-      })
-      .then(() => {
-        props.onFormSuccess?.();
-        form.reset();
-      })
-      .catch((error) => {
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-        if ("message" in error && typeof error.message === "string") {
-          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-          setFormError(error.message as string);
-          return;
-        } else {
-          setFormError(JSON.stringify(error));
-          console.error(error);
-        }
-      });
-  }
-
-  return (
-    <Form {...form}>
-      <form
-        // eslint-disable-next-line @typescript-eslint/no-misused-promises
-        onSubmit={form.handleSubmit(onSubmit)}
-        className="flex flex-col gap-2"
-      >
-        <Header level="h3" title="Name" />
-        <FormField
-          control={form.control}
-          name="modelName"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>Model Name</FormLabel>
-              <FormControl>
-                <AutoComplete
-                  {...field}
-                  options={
-                    modelNames.data?.map((model) => ({
-                      value: model,
-                      label: model,
-                    })) ?? []
-                  }
-                  placeholder=""
-                  onValueChange={(option) => field.onChange(option.value)}
-                  value={{ value: field.value, label: field.value }}
-                  disabled={false}
-                  createLabel="Create a new model name"
-                />
-              </FormControl>
-              <FormDescription>
-                The name of the model. This will be used to reference the model
-                in the API. You can track price changes of models by using the
-                same name and match pattern.
-              </FormDescription>
-              <FormMessage />
-            </FormItem>
-          )}
-        />
-        <Header level="h3" title="Scope" className="mt-3" />
-        <FormField
-          control={form.control}
-          name="matchPattern"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>Match pattern</FormLabel>
-              <FormControl>
-                <Input {...field} />
-              </FormControl>
-              <FormDescription>
-                Regular expression (Postgres syntax) to match ingested
-                generations (model attribute) to this model definition. For an
-                exact, case-insensitive match to a model name, use the
-                expression: (?i)^modelname$
-              </FormDescription>
-              <FormMessage />
-            </FormItem>
-          )}
-        />
-        <FormField
-          control={form.control}
-          name="startDate"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>Start date (UTC)</FormLabel>
-              <FormControl>
-                <DatePicker
-                  date={field.value}
-                  onChange={(date) => field.onChange(date)}
-                  clearable
-                />
-              </FormControl>
-              <FormDescription>
-                If set, the model will only be used for generations after this
-                date.
-              </FormDescription>
-              <FormMessage />
-            </FormItem>
-          )}
-        />
-        <Header level="h3" title="Pricing" className="mt-3" />
-        <FormField
-          control={form.control}
-          name="unit"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>Unit</FormLabel>
-              <Select onValueChange={field.onChange} defaultValue={field.value}>
-                <FormControl>
-                  <SelectTrigger>
-                    <SelectValue placeholder="Select a unit" />
-                  </SelectTrigger>
-                </FormControl>
-                <SelectContent>
-                  {Object.values(ModelUsageUnit).map((unit) => (
-                    <SelectItem value={unit} key={unit}>
-                      {unit}
-                    </SelectItem>
-                  ))}
-                </SelectContent>
-              </Select>
-              <FormDescription>
-                The unit of measurement for the model.
-              </FormDescription>
-              <FormMessage />
-            </FormItem>
-          )}
-        />
-        <div className="grid grid-cols-3 gap-2">
-          <FormField
-            control={form.control}
-            name="inputPrice"
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>
-                  Input price (USD per{" "}
-                  {form.getValues("unit").toLowerCase().replace(/s$/, "")})
-                </FormLabel>
-                <FormControl>
-                  <Input {...field} type="number" />
-                </FormControl>
-                {field.value !== null && field.value !== "" ? (
-                  <FormDescription>
-                    <ul className="font-mono text-xs">
-                      <li>
-                        {(parseFloat(field.value ?? "0") * 1000).toFixed(8)} USD
-                        / 1k {form.getValues("unit").toLowerCase()}
-                      </li>
-                      <li>
-                        {(parseFloat(field.value ?? "0") * 100_000).toFixed(8)}{" "}
-                        USD / 100k {form.getValues("unit").toLowerCase()}
-                      </li>
-                      <li>
-                        {(parseFloat(field.value ?? "0") * 1_000_000).toFixed(
-                          8,
-                        )}{" "}
-                        USD / 1M {form.getValues("unit").toLowerCase()}
-                      </li>
-                    </ul>
-                  </FormDescription>
-                ) : null}
-                <FormMessage />
-              </FormItem>
-            )}
-          />
-          <FormField
-            control={form.control}
-            name="outputPrice"
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>
-                  Output price (USD per{" "}
-                  {form.getValues("unit").toLowerCase().replace(/s$/, "")})
-                </FormLabel>
-                <FormControl>
-                  <Input {...field} type="number" />
-                </FormControl>
-                {field.value !== null && field.value !== "" ? (
-                  <FormDescription>
-                    <ul className="font-mono text-xs">
-                      <li>
-                        {(parseFloat(field.value ?? "0") * 1000).toFixed(8)} USD
-                        / 1k {form.getValues("unit").toLowerCase()}
-                      </li>
-                      <li>
-                        {(parseFloat(field.value ?? "0") * 100_000).toFixed(8)}{" "}
-                        USD / 100k {form.getValues("unit").toLowerCase()}
-                      </li>
-                      <li>
-                        {(parseFloat(field.value ?? "0") * 1_000_000).toFixed(
-                          8,
-                        )}{" "}
-                        USD / 1M {form.getValues("unit").toLowerCase()}
-                      </li>
-                    </ul>
-                  </FormDescription>
-                ) : null}
-                <FormMessage />
-              </FormItem>
-            )}
-          />
-          <FormField
-            control={form.control}
-            name="totalPrice"
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>
-                  Total price (USD per{" "}
-                  {form.getValues("unit").toLowerCase().replace(/s$/, "")})
-                </FormLabel>
-                <FormControl>
-                  <Input {...field} type="number" />
-                </FormControl>
-                <FormDescription>
-                  {field.value !== null && field.value !== "" ? (
-                    <ul className="mt-2 font-mono text-xs">
-                      <li>
-                        {(parseFloat(field.value ?? "0") * 1000).toFixed(8)} USD
-                        / 1k {form.getValues("unit").toLowerCase()}
-                      </li>
-                      <li>
-                        {(parseFloat(field.value ?? "0") * 100_000).toFixed(8)}{" "}
-                        USD / 100k {form.getValues("unit").toLowerCase()}
-                      </li>
-                      <li>
-                        {(parseFloat(field.value ?? "0") * 1_000_000).toFixed(
-                          8,
-                        )}{" "}
-                        USD / 1M {form.getValues("unit").toLowerCase()}
-                      </li>
-                    </ul>
-                  ) : (
-                    "Enter total price only if no separate input and output prices are provided."
-                  )}
-                </FormDescription>
-                <FormMessage />
-              </FormItem>
-            )}
-          />
-        </div>
-        <Header level="h3" title="Tokenization" className="mt-3" />
-        <FormField
-          control={form.control}
-          name="tokenizerId"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>Tokenizer</FormLabel>
-              <Select
-                onValueChange={(tokenizerId) => {
-                  field.onChange(tokenizerId);
-                  if (tokenizerId === "None") {
-                    form.setValue("tokenizerConfig", "{}");
-                  }
-                }}
-                defaultValue={field.value}
-              >
-                <FormControl>
-                  <SelectTrigger>
-                    <SelectValue placeholder="Select a unit" />
-                  </SelectTrigger>
-                </FormControl>
-                <SelectContent>
-                  {["openai", "claude", "None"].map((unit) => (
-                    <SelectItem value={unit} key={unit}>
-                      {unit}
-                    </SelectItem>
-                  ))}
-                </SelectContent>
-              </Select>
-              <FormDescription>
-                Optionally, Langfuse can tokenize the input and output of a
-                generation if no unit counts are ingested. This is useful for
-                e.g. streamed OpenAI completions. For details on the supported
-                tokenizers, see the{" "}
-                <Link
-                  href="https://langfuse.com/docs/model-usage-and-cost"
-                  className="underline"
-                  target="_blank"
-                >
-                  docs
-                </Link>
-                .
-              </FormDescription>
-              <FormMessage />
-            </FormItem>
-          )}
-        />
-        {form.watch("tokenizerId") !== "None" && (
-          <FormField
-            control={form.control}
-            name="tokenizerConfig"
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>Tokenizer Config</FormLabel>
-                <JsonEditor
-                  defaultValue={field.value}
-                  onChange={field.onChange}
-                />
-                <FormDescription>
-                  The config for the tokenizer. Required for openai. See the{" "}
-                  <Link
-                    href="https://langfuse.com/docs/model-usage-and-cost"
-                    className="underline"
-                    target="_blank"
-                  >
-                    docs
-                  </Link>{" "}
-                  for details.
-                </FormDescription>
-                <FormMessage />
-              </FormItem>
-            )}
-          />
-        )}
-        <Button
-          type="submit"
-          loading={createModelMutation.isLoading}
-          className="mt-6"
-        >
-          Save
-        </Button>
-      </form>
-      {formError ? (
-        <p className="text-red text-center">
-          <span className="font-bold">Error:</span> {formError}
-        </p>
-      ) : null}
-    </Form>
-  );
-};

web/src/features/models/components/PriceBreakdownTooltip.tsx

@@ -0,0 +1,114 @@
+import Decimal from "decimal.js";
+import { InfoIcon } from "lucide-react";
+import { useMemo, useState } from "react";
+
+import { type RowHeight } from "@/src/components/table/data-table-row-height-switch";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/src/components/ui/tooltip";
+import { usePriceUnitMultiplier } from "@/src/features/models/hooks/usePriceUnitMultiplier";
+import { getMaxDecimals } from "@/src/features/models/utils";
+import { type PriceUnit } from "@/src/features/models/validation";
+
+export const PriceBreakdownTooltip = ({
+  modelName,
+  prices,
+  priceUnit,
+  rowHeight,
+}: {
+  modelName: string;
+  prices?: Record<string, number>;
+  priceUnit: PriceUnit;
+  rowHeight: RowHeight;
+}) => {
+  const [isOpen, setIsOpen] = useState(false);
+  const { priceUnitMultiplier } = usePriceUnitMultiplier();
+
+  const maxDecimals = useMemo(
+    () =>
+      Math.max(
+        ...Object.values(prices ?? {}).map((price) => {
+          return getMaxDecimals(price, priceUnitMultiplier);
+        }),
+      ),
+    [prices, priceUnitMultiplier],
+  );
+
+  if (!prices) return null;
+
+  return (
+    <>
+      {Object.keys(prices).length === 0 ? (
+        <p>No prices</p>
+      ) : Object.keys(prices).length <= (rowHeight === "m" ? 4 : 2) ? (
+        <div className="grid w-full grid-cols-[2fr,3fr] gap-x-2">
+          {Object.entries(prices).map(([type, price]) => (
+            <>
+              <span
+                key={`${type}-label`}
+                className="truncate font-mono text-xs font-medium"
+                title={type}
+              >
+                {type}
+              </span>
+              <span
+                key={`${type}-price`}
+                className="text-left font-mono text-xs font-medium tabular-nums"
+              >
+                $
+                {new Decimal(price)
+                  .mul(priceUnitMultiplier)
+                  .toFixed(maxDecimals)}
+              </span>
+            </>
+          ))}
+        </div>
+      ) : (
+        <TooltipProvider>
+          <Tooltip open={isOpen} onOpenChange={setIsOpen}>
+            <TooltipTrigger
+              className="flex cursor-pointer items-center gap-2 pr-[1rem] text-xs"
+              onClick={() => setIsOpen(!isOpen)}
+            >
+              <InfoIcon className="h-3 w-3" />
+              {Object.keys(prices).length} prices set
+            </TooltipTrigger>
+            <TooltipContent className="min-w-[16rem] grow p-4">
+              <div className="flex flex-col gap-4">
+                <div className="flex flex-col gap-1">
+                  <span className="font-semibold">Price breakdown</span>
+                  <span className="font-mono text-xs font-medium">
+                    {modelName}
+                  </span>
+                </div>
+                <div className="flex flex-col gap-2">
+                  <div className="flex justify-between font-mono text-xs font-semibold">
+                    <span className="mr-4">Usage Type</span>
+                    <span>Price {priceUnit}</span>
+                  </div>
+                  {Object.entries(prices).map(([usageType, price]) => (
+                    <div
+                      key={usageType}
+                      className="flex justify-between font-mono text-xs"
+                    >
+                      <span className="mr-4">{usageType}</span>
+                      <span>
+                        {"$" +
+                          new Decimal(price)
+                            .mul(priceUnitMultiplier)
+                            .toFixed(maxDecimals)}
+                      </span>
+                    </div>
+                  ))}
+                </div>
+              </div>
+            </TooltipContent>
+          </Tooltip>
+        </TooltipProvider>
+      )}
+    </>
+  );
+};

web/src/features/models/components/PricePreview.tsx

@@ -0,0 +1,81 @@
+import Decimal from "decimal.js";
+
+import { PriceMapSchema } from "@/src/features/models/validation";
+import { getMaxDecimals } from "@/src/features/models/utils";
+
+export function PricePreview({
+  prices,
+}: {
+  prices: Record<string, number | undefined>;
+}) {
+  const parsedPrices = PriceMapSchema.safeParse(prices);
+
+  const getMaxDecimalsForPriceGroup = (
+    price: number | undefined,
+    multiplier: number,
+  ) => {
+    return price != null
+      ? Math.max(
+          ...Object.values(prices).map((price) => {
+            return getMaxDecimals(price, multiplier);
+          }),
+        )
+      : 0;
+  };
+
+  return (
+    <div className="rounded-lg border border-border bg-muted/30 p-4">
+      <div className="flex flex-col gap-3">
+        <div className="flex items-center gap-2">
+          <h4 className="text-sm font-medium text-muted-foreground">
+            Price Preview
+          </h4>
+        </div>
+
+        {parsedPrices.success ? (
+          <div className="space-y-2">
+            <div className="grid grid-cols-[2fr_1fr_1fr_1fr] gap-2 border-b border-border pb-2 text-xs font-medium text-muted-foreground">
+              <span>Usage Type</span>
+              <span className="text-right">per unit</span>
+              <span className="text-right">per 1K</span>
+              <span className="text-right">per 1M</span>
+            </div>
+
+            {Object.entries(parsedPrices.data)
+              .filter((entry): entry is [string, number] => Boolean(entry[1]))
+              .map(([usageType, price]) => (
+                <div
+                  key={usageType}
+                  className="grid grid-cols-[2fr_1fr_1fr_1fr] gap-2 rounded px-1 py-0.5 text-xs text-muted-foreground"
+                >
+                  <span className="break-all font-medium">{usageType}</span>
+                  <span className="text-right font-mono">
+                    $
+                    {new Decimal(price).toFixed(
+                      getMaxDecimalsForPriceGroup(price, 1),
+                    )}
+                  </span>
+                  <span className="text-right font-mono">
+                    $
+                    {new Decimal(price)
+                      .mul(1000)
+                      .toFixed(getMaxDecimalsForPriceGroup(price, 1000))}
+                  </span>
+                  <span className="text-right font-mono">
+                    $
+                    {new Decimal(price)
+                      .mul(1000000)
+                      .toFixed(getMaxDecimalsForPriceGroup(price, 1000000))}
+                  </span>
+                </div>
+              ))}
+          </div>
+        ) : (
+          <div className="rounded-md bg-destructive/10 p-3 text-sm text-destructive">
+            Invalid price entries. Please check your input format.
+          </div>
+        )}
+      </div>
+    </div>
+  );
+}

web/src/features/models/components/PriceUnitSelector.tsx

@@ -0,0 +1,48 @@
+import { ChevronDownIcon } from "lucide-react";
+
+import { Button } from "@/src/components/ui/button";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { PriceUnit } from "@/src/features/models/validation";
+import { usePriceUnitMultiplier } from "@/src/features/models/hooks/usePriceUnitMultiplier";
+
+export const PriceUnitSelector = () => {
+  const { priceUnit, setPriceUnit } = usePriceUnitMultiplier();
+
+  return (
+    <Popover>
+      <PopoverTrigger asChild>
+        <Button size="icon" variant="ghost">
+          <ChevronDownIcon className="h-4 w-4" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent className="w-[200px] p-0">
+        <Select
+          value={priceUnit}
+          onValueChange={(value: PriceUnit) => setPriceUnit(value)}
+        >
+          <SelectTrigger className="w-full">
+            <SelectValue placeholder="Select unit" />
+          </SelectTrigger>
+          <SelectContent>
+            {Object.values(PriceUnit).map((unit) => (
+              <SelectItem key={unit} value={unit}>
+                {unit}
+              </SelectItem>
+            ))}
+          </SelectContent>
+        </Select>
+      </PopoverContent>
+    </Popover>
+  );
+};

web/src/features/models/components/UpsertModelFormDrawer.tsx

@@ -0,0 +1,486 @@
+import { MinusCircle, PlusCircle, X } from "lucide-react";
+import Link from "next/link";
+import { useEffect, useState } from "react";
+import { useForm } from "react-hook-form";
+import * as z from "zod";
+
+import { JsonEditor } from "@/src/components/json-editor";
+import { Button } from "@/src/components/ui/button";
+import {
+  Drawer,
+  DrawerContent,
+  DrawerDescription,
+  DrawerFooter,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from "@/src/components/ui/drawer";
+import {
+  Form,
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import { Input } from "@/src/components/ui/input";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import {
+  type FormUpsertModel,
+  FormUpsertModelSchema,
+  type GetModelResult,
+} from "@/src/features/models/validation";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { api } from "@/src/utils/api";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { useRouter } from "next/router";
+
+import { PricePreview } from "./PricePreview";
+import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+
+type UpsertModelDrawerProps =
+  | {
+      action: "create";
+      children: React.ReactNode;
+      projectId: string;
+      prefilledModelData?: {
+        modelName?: string;
+        prices?: Record<string, number>;
+      };
+      className?: string;
+    }
+  | {
+      action: "edit" | "clone";
+      children: React.ReactNode;
+      projectId: string;
+      modelData: GetModelResult;
+      className?: string;
+    };
+
+export const UpsertModelFormDrawer = ({
+  children,
+  ...props
+}: UpsertModelDrawerProps) => {
+  const capture = usePostHogClientCapture();
+  const router = useRouter();
+  const [formError, setFormError] = useState<string | null>(null);
+  const utils = api.useUtils();
+  const [open, setOpen] = useState(false);
+
+  let defaultValues: FormUpsertModel;
+  if (props.action !== "create") {
+    defaultValues = {
+      modelName: props.modelData.modelName,
+      matchPattern: props.modelData.matchPattern,
+      tokenizerId: props.modelData.tokenizerId,
+      tokenizerConfig: JSON.stringify(props.modelData.tokenizerConfig ?? {}),
+      prices: props.modelData.prices,
+    };
+  } else {
+    defaultValues = {
+      modelName: props.prefilledModelData?.modelName ?? "",
+      matchPattern: props.prefilledModelData?.modelName
+        ? `(?i)^(${props.prefilledModelData?.modelName})$`
+        : "",
+      tokenizerId: null,
+      tokenizerConfig: null,
+      prices: props.prefilledModelData?.prices ?? {
+        input: 0.000001,
+        output: 0.000002,
+      },
+    };
+  }
+
+  const form = useForm<FormUpsertModel>({
+    resolver: zodResolver(
+      props.action === "edit"
+        ? FormUpsertModelSchema.omit({ modelName: true }).extend({
+            modelName: z.string().default(props.modelData.modelName),
+          })
+        : FormUpsertModelSchema,
+    ),
+    defaultValues,
+  });
+  const modelName = form.watch("modelName");
+  const matchPattern = form.watch("matchPattern");
+  const tokenizerId = form.watch("tokenizerId");
+
+  // prefill match pattern if model name changes
+  useEffect(() => {
+    const getRegexString = (modelName: string) => `(?i)^(${modelName})$`;
+
+    if (
+      modelName &&
+      (!matchPattern ||
+        matchPattern === `(?i)^(${modelName.slice(0, -1)})$` ||
+        matchPattern === `(?i)^(${modelName})$`)
+    ) {
+      form.setValue("matchPattern", getRegexString(modelName));
+    }
+  }, [modelName, matchPattern, form]);
+
+  const upsertModelMutation = api.models.upsert.useMutation({
+    onSuccess: (upsertedModel) => {
+      utils.models.invalidate();
+      form.reset();
+      setOpen(false);
+      showSuccessToast({
+        title: `Model ${props.action === "edit" ? "updated" : "created"}`,
+        description: `The model '${upsertedModel.modelName}' has been successfully ${props.action === "edit" ? "updated" : "created"}. New generations will use these model prices.`,
+      });
+      router.push(`/project/${props.projectId}/models/${upsertedModel.id}`);
+    },
+    onError: (error) => setFormError(error.message),
+  });
+
+  const onSubmit = async (values: FormUpsertModel) => {
+    capture("models:new_form_submit");
+
+    await upsertModelMutation
+      .mutateAsync({
+        modelId: props.action === "edit" ? props.modelData.id : null,
+        projectId: props.projectId,
+        modelName: values.modelName,
+        matchPattern: values.matchPattern,
+        prices: values.prices,
+        tokenizerId: values.tokenizerId,
+        tokenizerConfig:
+          values.tokenizerConfig &&
+          typeof JSON.parse(values.tokenizerConfig) === "object"
+            ? (JSON.parse(values.tokenizerConfig) as Record<string, number>)
+            : undefined,
+      })
+      .catch((error) => {
+        setFormError(error.message);
+      });
+  };
+
+  return (
+    <Drawer
+      open={open}
+      onOpenChange={(open) => {
+        if (!open) return; // Only allow closing via cancel key
+        setOpen(open);
+      }}
+      dismissible={false}
+      onClose={() => {
+        form.reset();
+        setFormError(null);
+      }}
+    >
+      <DrawerTrigger
+        asChild
+        onClick={() => setOpen(true)}
+        className={props.className}
+        title={
+          props.action === "create"
+            ? "Create model definition"
+            : "Edit model definition"
+        }
+      >
+        {children}
+      </DrawerTrigger>
+      <DrawerContent>
+        <DrawerHeader>
+          <div className="flex items-center justify-between">
+            <DrawerTitle>
+              {props.action === "create"
+                ? "Create Model"
+                : props.action === "clone"
+                  ? "Clone Model"
+                  : "Edit Model"}
+            </DrawerTitle>
+            <Button
+              variant="ghost"
+              size="icon"
+              onClick={() => setOpen(false)}
+              type="button"
+            >
+              <X size={20} />
+            </Button>
+          </div>
+          <DrawerDescription>
+            {props.action === "edit"
+              ? props.modelData.modelName
+              : props.action === "create"
+                ? "Create a new model configuration to track generation costs."
+                : null}
+          </DrawerDescription>
+        </DrawerHeader>
+        <Form {...form}>
+          <form
+            // eslint-disable-next-line @typescript-eslint/no-misused-promises
+            onSubmit={form.handleSubmit(onSubmit)}
+            className="flex h-full max-h-[100vh] flex-col gap-6 overflow-y-auto p-4 pt-0"
+          >
+            <FormField
+              control={form.control}
+              name="modelName"
+              disabled={props.action === "edit"}
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Model Name</FormLabel>
+                  <FormDescription>
+                    The name of the model. This will be used to reference the
+                    model in the API. You can track price changes of models by
+                    using the same name and match pattern.
+                  </FormDescription>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <FormField
+              control={form.control}
+              name="matchPattern"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Match pattern</FormLabel>
+                  <FormDescription>
+                    Regular expression (Postgres syntax) to match ingested
+                    generations (model attribute) to this model definition. For
+                    an exact, case-insensitive match to a model name, use the
+                    expression: (?i)^(modelname)$
+                  </FormDescription>
+                  <FormControl>
+                    <Input {...field} />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <FormField
+              control={form.control}
+              name="prices"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel className="flex items-center gap-2">
+                    Prices
+                  </FormLabel>
+                  <FormDescription>
+                    Set prices per usage type for this model. Usage types must
+                    exactly match the keys of the ingested usage details.
+                  </FormDescription>
+                  <span className="flex flex-col gap-2">
+                    <FormDescription>
+                      Prefill usage types from template:
+                    </FormDescription>
+                    <span className="flex gap-2">
+                      <Button
+                        type="button"
+                        variant="outline"
+                        size="sm"
+                        onClick={() => {
+                          field.onChange({
+                            input: 0,
+                            output: 0,
+                            input_cached_tokens: 0,
+                            output_reasoning_tokens: 0,
+                            ...field.value,
+                          });
+                        }}
+                      >
+                        OpenAI
+                      </Button>
+                      <Button
+                        type="button"
+                        variant="outline"
+                        size="sm"
+                        onClick={() => {
+                          field.onChange({
+                            input: 0,
+                            input_tokens: 0,
+                            output: 0,
+                            output_tokens: 0,
+                            cache_creation_input_tokens: 0,
+                            cache_read_input_tokens: 0,
+                            ...field.value,
+                          });
+                        }}
+                      >
+                        Anthropic
+                      </Button>
+                    </span>
+                  </span>
+                  <FormControl>
+                    <span className="flex flex-col gap-2">
+                      <FormDescription className="grid grid-cols-2 gap-1">
+                        <span>Usage type</span>
+                        <span>Price</span>
+                      </FormDescription>
+                      {Object.entries(field.value).map(
+                        ([key, value], index) => (
+                          <div key={index} className="grid grid-cols-2 gap-1">
+                            <Input
+                              placeholder="Key (e.g. input, output)"
+                              value={key}
+                              onChange={(e) => {
+                                const newPrices = { ...field.value };
+                                const oldValue = newPrices[key];
+                                delete newPrices[key];
+                                newPrices[e.target.value] = oldValue;
+                                field.onChange(newPrices);
+                              }}
+                            />
+                            <div className="flex gap-1">
+                              <Input
+                                type="number"
+                                placeholder="Price per unit"
+                                value={value}
+                                step="0.000001"
+                                onChange={(e) => {
+                                  field.onChange({
+                                    ...field.value,
+                                    [key]: parseFloat(e.target.value),
+                                  });
+                                }}
+                              />
+                              <Button
+                                type="button"
+                                variant="outline"
+                                title="Remove price"
+                                size="icon"
+                                onClick={() => {
+                                  const newPrices = { ...field.value };
+                                  delete newPrices[key];
+                                  field.onChange(newPrices);
+                                }}
+                              >
+                                <MinusCircle className="h-4 w-4" />
+                              </Button>
+                            </div>
+                          </div>
+                        ),
+                      )}
+                      <Button
+                        type="button"
+                        variant="ghost"
+                        onClick={() => {
+                          field.onChange({
+                            ...field.value,
+                            new_usage_type: 0.000001,
+                          });
+                        }}
+                        className="flex items-center gap-1"
+                      >
+                        <PlusCircle className="h-4 w-4" />
+                        <span>Add Price</span>
+                      </Button>
+                      <PricePreview prices={field.value} />
+                    </span>
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <FormField
+              control={form.control}
+              name="tokenizerId"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>Tokenizer</FormLabel>
+                  <Select
+                    onValueChange={(tokenizerId) => {
+                      field.onChange(tokenizerId);
+                      if (tokenizerId === "None") {
+                        form.setValue("tokenizerConfig", "{}");
+                      }
+                    }}
+                    defaultValue={field.value ?? "None"}
+                  >
+                    <FormControl>
+                      <SelectTrigger>
+                        <SelectValue placeholder="Select a unit" />
+                      </SelectTrigger>
+                    </FormControl>
+                    <SelectContent>
+                      {["openai", "claude", "None"].map((unit) => (
+                        <SelectItem value={unit} key={unit}>
+                          {unit}
+                        </SelectItem>
+                      ))}
+                    </SelectContent>
+                  </Select>
+                  <FormDescription>
+                    Optionally, Langfuse can tokenize the input and output of a
+                    generation if no unit counts are ingested. This is useful
+                    for e.g. streamed OpenAI completions. For details on the
+                    supported tokenizers, see the{" "}
+                    <Link
+                      href="https://langfuse.com/docs/model-usage-and-cost"
+                      className="underline"
+                      target="_blank"
+                    >
+                      docs
+                    </Link>
+                    .
+                  </FormDescription>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            {tokenizerId && tokenizerId !== "None" && (
+              <FormField
+                control={form.control}
+                name="tokenizerConfig"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Tokenizer Config</FormLabel>
+                    <JsonEditor
+                      defaultValue={field.value ?? "{}"}
+                      onChange={field.onChange}
+                    />
+                    <FormDescription>
+                      The config for the tokenizer. Required for openai. See the{" "}
+                      <Link
+                        href="https://langfuse.com/docs/model-usage-and-cost"
+                        className="underline"
+                        target="_blank"
+                      >
+                        docs
+                      </Link>{" "}
+                      for details.
+                    </FormDescription>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+            )}
+            <DrawerFooter className="flex-row gap-2">
+              <Button
+                type="button"
+                variant="outline"
+                onClick={() => setOpen(false)}
+                className="w-full"
+              >
+                Cancel
+              </Button>
+
+              <Button
+                type="submit"
+                className="w-full"
+                loading={upsertModelMutation.isLoading}
+              >
+                Submit
+              </Button>
+            </DrawerFooter>
+          </form>
+          {formError ? (
+            <p className="my-2 text-center text-sm font-medium text-destructive">
+              <span className="font-semibold">Error:</span> {formError}
+            </p>
+          ) : null}
+        </Form>
+      </DrawerContent>
+    </Drawer>
+  );
+};

web/src/features/models/hooks/usePriceUnitMultiplier.ts

@@ -0,0 +1,20 @@
+import { useMemo } from "react";
+
+import useLocalStorage from "@/src/components/useLocalStorage";
+import { PriceUnit } from "@/src/features/models/validation";
+
+export const multiplierMap: Record<PriceUnit, number> = {
+  [PriceUnit.PerUnit]: 1,
+  [PriceUnit.Per1KUnits]: 1e3,
+  [PriceUnit.Per1MUnits]: 1e6,
+};
+
+export const usePriceUnitMultiplier = () => {
+  const [priceUnit, setPriceUnit] = useLocalStorage<PriceUnit>(
+    "priceUnit",
+    PriceUnit.PerUnit,
+  );
+  const multiplier = useMemo(() => multiplierMap[priceUnit], [priceUnit]);
+
+  return { priceUnit, setPriceUnit, priceUnitMultiplier: multiplier };
+};

web/src/features/models/utils.ts

@@ -0,0 +1,14 @@
+import Decimal from "decimal.js";
+
+export const getMaxDecimals = (
+  value: number | undefined,
+  scaleMultiplier: number = 1,
+) => {
+  return (
+    new Decimal(value ?? 0)
+      .mul(scaleMultiplier)
+      .toFixed(12)
+      .split(".")[1]
+      ?.replace(/0+$/, "").length ?? 0
+  );
+};

web/src/features/models/validation.ts

@@ -0,0 +1,77 @@
+import { z } from "zod";
+
+export const UsageTypeSchema = z.string().regex(/^[a-zA-Z0-9_-]+$/);
+export const PriceSchema = z.number().nonnegative();
+export const TokenizerSchema = z.enum(["openai", "claude"]).nullish();
+export const PriceMapSchema = z
+  .record(UsageTypeSchema, PriceSchema.optional())
+  .transform((obj) => {
+    return Object.fromEntries(
+      Object.entries(obj).filter(([_, value]) => Boolean(value)),
+    );
+  })
+  .pipe(z.record(UsageTypeSchema, PriceSchema));
+
+export const GetModelResultSchema = z.object({
+  id: z.string(),
+  projectId: z.string().nullable(),
+  modelName: z.string(),
+  matchPattern: z.string(),
+  tokenizerConfig: z
+    .record(z.union([z.string(), z.coerce.number()]))
+    .nullable(),
+  tokenizerId: TokenizerSchema,
+  prices: PriceMapSchema,
+});
+
+export type GetModelResult = z.infer<typeof GetModelResultSchema>;
+
+export const UpsertModelSchema = z.object({
+  modelId: z.string().nullable(),
+  projectId: z.string(),
+  modelName: z.string().min(1),
+  matchPattern: z.string().min(1),
+  tokenizerId: z
+    .enum(["openai", "claude", "None"])
+    .nullish()
+    .transform((value) => {
+      return value === "None" ? null : value;
+    })
+    .pipe(TokenizerSchema.nullish()),
+  tokenizerConfig: z
+    .record(z.union([z.string(), z.coerce.number()]))
+    .optional(),
+  prices: PriceMapSchema,
+});
+export type UpsertModel = z.infer<typeof UpsertModelSchema>;
+
+export const FormUpsertModelSchema = z.object({
+  modelName: z.string().min(1),
+  matchPattern: z.string().min(1),
+  tokenizerId: z.enum(["openai", "claude", "None"]).nullish(),
+  tokenizerConfig: z
+    .string()
+    .refine(
+      (value) => {
+        try {
+          JSON.parse(value);
+          return true;
+        } catch {
+          return false;
+        }
+      },
+      {
+        message: "Tokenizer config needs to be valid JSON",
+      },
+    )
+    .transform((value) => (value === "{}" ? undefined : value))
+    .nullish(),
+  prices: PriceMapSchema,
+});
+export type FormUpsertModel = z.infer<typeof FormUpsertModelSchema>;
+
+export enum PriceUnit {
+  PerUnit = "per unit",
+  Per1KUnits = "per 1K units",
+  Per1MUnits = "per 1M units",
+}

web/src/features/prompts/components/NewPromptForm/index.tsx

@@ -48,6 +48,7 @@ import { PRODUCTION_LABEL } from "@/src/features/prompts/constants";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
 import { useQueryParam } from "use-query-params";
+import { Switch } from "@/src/components/ui/switch";
 
 type NewPromptFormProps = {
   initialPrompt?: Prompt | null;
@@ -61,6 +62,7 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
   const [formError, setFormError] = useState<string | null>(null);
   const { playgroundCache } = usePlaygroundCache();
   const [initialMessages, setInitialMessages] = useState<unknown>([]);
+  const [showJsonEditor, setShowJsonEditor] = useState(false);
 
   const utils = api.useUtils();
   const capture = usePostHogClientCapture();
@@ -250,7 +252,24 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
         {/* Prompt content field - text vs. chat */}
         <>
           <FormItem>
-            <FormLabel>Prompt</FormLabel>
+            <FormLabel className="flex flex-row items-center justify-between">
+              <div>Prompt</div>
+              {form.watch("type") === PromptType.Text ? (
+                <div className="flex flex-row items-center">
+                  <p className="mr-1 text-xs text-muted-foreground">
+                    JSON editor
+                  </p>
+
+                  <Switch
+                    checked={showJsonEditor}
+                    className={
+                      showJsonEditor ? "data-[state=checked]:bg-dark-green" : ""
+                    }
+                    onCheckedChange={setShowJsonEditor}
+                  />
+                </div>
+              ) : null}
+            </FormLabel>
             <Tabs
               value={form.watch("type")}
               onValueChange={(e) => {
@@ -288,10 +307,18 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
                   render={({ field }) => (
                     <>
                       <FormControl>
-                        <Textarea
-                          {...field}
-                          className="min-h-[200px] flex-1 font-mono text-xs"
-                        />
+                        {showJsonEditor ? (
+                          <JsonEditor
+                            defaultValue={field.value}
+                            onChange={field.onChange}
+                            editable
+                          />
+                        ) : (
+                          <Textarea
+                            {...field}
+                            className="min-h-[200px] flex-1 font-mono text-xs"
+                          />
+                        )}
                       </FormControl>
                       <FormMessage />
                     </>

web/src/features/scores/adapters/index.ts

@@ -0,0 +1,49 @@
+import {
+  transformCategoricalScoresToChartData,
+  uniqueAndSort,
+} from "@/src/features/dashboard/lib/score-analytics-utils";
+import {
+  type ChartBin,
+  type ChartData,
+  type TimeseriesDataTransformer,
+} from "@/src/features/scores/types";
+import { type DatabaseRow } from "@/src/server/api/services/queryBuilder";
+import { type DashboardDateRangeAggregationOption } from "@/src/utils/date-range-utils";
+
+export class DashboardCategoricalScoreAdapter
+  implements TimeseriesDataTransformer
+{
+  constructor(
+    private data: DatabaseRow[],
+    private timestamp: string,
+    private agg?: DashboardDateRangeAggregationOption,
+  ) {}
+
+  toChartData(): ChartData {
+    const { chartData, chartLabels } = transformCategoricalScoresToChartData(
+      this.data,
+      this.timestamp,
+      this.agg,
+    );
+    return { chartData, chartLabels: uniqueAndSort(chartLabels) };
+  }
+}
+
+export class CompareViewAdapter implements TimeseriesDataTransformer {
+  constructor(
+    private runMetrics: Map<
+      string,
+      { chartData: ChartBin[]; chartLabels: string[] }
+    >,
+    private key: string,
+  ) {}
+
+  toChartData(): ChartData {
+    return {
+      chartData: this.runMetrics.get(this.key)?.chartData ?? [],
+      chartLabels: uniqueAndSort(
+        this.runMetrics.get(this.key)?.chartLabels ?? [],
+      ),
+    };
+  }
+}

web/src/features/scores/components/AnnotateDrawerContent.tsx

@@ -61,11 +61,11 @@ import { getDefaultScoreData } from "@/src/features/scores/lib/getDefaultScoreDa
 import { ToggleGroup, ToggleGroupItem } from "@/src/components/ui/toggle-group";
 import Header from "@/src/components/layouts/header";
 import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";
-import { CommandItem } from "@/src/components/ui/command";
 import { useRouter } from "next/router";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { cn } from "@/src/utils/tailwind";
 import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
+import { DropdownMenuItem } from "@/src/components/ui/dropdown-menu";
 
 const AnnotationScoreDataSchema = z.object({
   name: z.string(),
@@ -712,7 +712,7 @@ export function AnnotateDrawerContent({
                   key: field.configId as string,
                 }))}
               controlButtons={
-                <CommandItem
+                <DropdownMenuItem
                   onSelect={() => {
                     capture("score_configs:manage_configs_item_click", {
                       type: type,
@@ -722,7 +722,7 @@ export function AnnotateDrawerContent({
                   }}
                 >
                   Manage score configs
-                </CommandItem>
+                </DropdownMenuItem>
               }
             />
           </div>

web/src/features/scores/components/ScoreChart.tsx

@@ -0,0 +1,80 @@
+import { compactNumberFormatter } from "@/src/utils/numbers";
+import { getColorsForCategories } from "@/src/features/dashboard/utils/getColorsForCategories";
+import { isEmptyChart } from "@/src/features/dashboard/lib/score-analytics-utils";
+import { BarChart, LineChart } from "@tremor/react";
+import { NoDataOrLoading } from "@/src/components/NoDataOrLoading";
+import { Card } from "@/src/components/ui/card";
+import { type ChartBin } from "@/src/features/scores/types";
+import { cn } from "@/src/utils/tailwind";
+
+export function CategoricalChart(props: {
+  chartData: ChartBin[];
+  chartLabels: string[];
+  isLoading?: boolean;
+  stack?: boolean;
+  showXAxis?: boolean;
+  className?: string;
+  chartClass?: string;
+}) {
+  const barCategoryGap = (chartLength: number): string => {
+    if (chartLength > 7) return "10%";
+    if (chartLength > 5) return "20%";
+    if (chartLength > 3) return "30%";
+    else return "40%";
+  };
+  const colors = getColorsForCategories(props.chartLabels);
+
+  return isEmptyChart({ data: props.chartData }) ? (
+    <NoDataOrLoading
+      isLoading={props.isLoading ?? false}
+      className={props.chartClass}
+    />
+  ) : (
+    <Card
+      className={cn("w-full rounded-tremor-default border", props.className)}
+    >
+      <BarChart
+        className={cn("mt-4", props.chartClass)}
+        data={props.chartData}
+        index="binLabel"
+        categories={props.chartLabels}
+        colors={colors}
+        valueFormatter={(number: number) =>
+          Intl.NumberFormat("en-US").format(number).toString()
+        }
+        yAxisWidth={48}
+        barCategoryGap={barCategoryGap(props.chartData.length)}
+        stack={props.stack ?? true}
+        showXAxis={props.showXAxis ?? true}
+      />
+    </Card>
+  );
+}
+
+export function NumericChart(props: {
+  chartData: ChartBin[];
+  chartLabels: string[];
+  index: string;
+}) {
+  const colors = getColorsForCategories(props.chartLabels);
+
+  return isEmptyChart({ data: props.chartData }) ? (
+    <NoDataOrLoading isLoading={false} />
+  ) : (
+    <Card className="h-full w-full rounded-tremor-default border">
+      <LineChart
+        className="h-full"
+        data={props.chartData}
+        index={props.index}
+        categories={props.chartLabels}
+        colors={colors}
+        valueFormatter={compactNumberFormatter}
+        noDataText="No data"
+        showAnimation={true}
+        onValueChange={() => {}}
+        enableLegendSlider={true}
+        showXAxis={false}
+      />
+    </Card>
+  );
+}

web/src/features/scores/components/TimeseriesChart.tsx

@@ -0,0 +1,46 @@
+import {
+  CategoricalChart,
+  NumericChart,
+} from "@/src/features/scores/components/ScoreChart";
+import { type TimeseriesChartProps } from "@/src/features/scores/types";
+
+function ChartWrapper(props: { title: string; children: React.ReactNode }) {
+  return (
+    <div className="mb-2 flex w-[80%] flex-none flex-col overflow-hidden md:w-[45%]">
+      <div className="shrink-0 text-sm font-medium">{props.title}</div>
+      {props.children}
+    </div>
+  );
+}
+
+export function TimeseriesChart({
+  chartData,
+  chartLabels,
+  title,
+  type,
+  index,
+}: TimeseriesChartProps) {
+  const chartIndex = index ?? "binLabel";
+
+  return (
+    <ChartWrapper title={title}>
+      <div className="mt-2 min-h-0 flex-1">
+        {type === "categorical" ? (
+          <CategoricalChart
+            chartLabels={chartLabels}
+            chartData={chartData}
+            className="h-full"
+            chartClass="h-full mt-0"
+            showXAxis={chartData.length < 3}
+          />
+        ) : (
+          <NumericChart
+            chartLabels={chartLabels}
+            chartData={chartData}
+            index={chartIndex}
+          />
+        )}
+      </div>
+    </ChartWrapper>
+  );
+}

web/src/features/scores/components/multi-select-key-values.tsx

@@ -1,23 +1,21 @@
 import * as React from "react";
-import { Archive, Check, ChevronDown } from "lucide-react";
+import { Archive, ChevronDown, Component, Search } from "lucide-react";
 
 import { cn } from "@/src/utils/tailwind";
 import { Badge } from "@/src/components/ui/badge";
 import { Button } from "@/src/components/ui/button";
+import { Input } from "@/src/components/ui/input";
 import {
-  Command,
-  CommandEmpty,
-  CommandGroup,
-  CommandInput,
-  CommandItem,
-  CommandList,
-  CommandSeparator,
-} from "@/src/components/ui/command";
-import {
-  Popover,
-  PopoverContent,
-  PopoverTrigger,
-} from "@/src/components/ui/popover";
+  DropdownMenu,
+  DropdownMenuCheckboxItem,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuSeparator,
+  DropdownMenuSub,
+  DropdownMenuSubContent,
+  DropdownMenuSubTrigger,
+  DropdownMenuTrigger,
+} from "@/src/components/ui/dropdown-menu";
 import { Separator } from "@/src/components/ui/separator";
 
 type MultiSelectOptions = {
@@ -28,6 +26,34 @@ type MultiSelectOptions = {
   isArchived?: boolean;
 };
 
+type MultiSelectGroup = {
+  label: string;
+  options: MultiSelectOptions[];
+};
+
+type MultiSelectKeyValuesProps<
+  T extends { key: string; value: string } | string,
+> = {
+  values: T[];
+  onValueChange: (
+    values: T[],
+    changedValue?: string,
+    selectedKeys?: Set<string>,
+  ) => void;
+  options: MultiSelectOptions[] | readonly MultiSelectOptions[];
+  title?: string;
+  placeholder?: string;
+  groupedOptions?: MultiSelectGroup[];
+  className?: string;
+  disabled?: boolean;
+  items?: string;
+  align?: "center" | "end" | "start";
+  controlButtons?: React.ReactNode;
+  hideClearButton?: boolean;
+  iconLeft?: React.ReactNode;
+  iconRight?: React.ReactNode;
+};
+
 export function MultiSelectKeyValues<
   T extends { key: string; value: string } | string,
 >({
@@ -36,29 +62,19 @@ export function MultiSelectKeyValues<
   values,
   onValueChange,
   options,
+  groupedOptions,
   className,
   disabled,
   items = "items",
   align = "center",
   controlButtons,
   hideClearButton = false,
-}: {
-  title?: string;
-  placeholder?: string;
-  values: T[];
-  onValueChange: (
-    values: T[],
-    changedValue?: string,
-    selectedKeys?: Set<string>,
-  ) => void;
-  options: MultiSelectOptions[] | readonly MultiSelectOptions[];
-  className?: string;
-  disabled?: boolean;
-  items?: string;
-  align?: "center" | "end" | "start";
-  controlButtons?: React.ReactNode;
-  hideClearButton?: boolean;
-}) {
+  iconLeft,
+  iconRight,
+}: MultiSelectKeyValuesProps<T>) {
+  const [isOpen, setIsOpen] = React.useState(false);
+  const [searchQuery, setSearchQuery] = React.useState("");
+
   const selectedValueKeys = new Set(
     values.map((value) => (typeof value === "string" ? value : value.key)),
   );
@@ -69,7 +85,11 @@ export function MultiSelectKeyValues<
       return Array.from(selectedValueKeys) as T[];
     }
 
-    return options
+    const allOptions = groupedOptions
+      ? groupedOptions.flatMap((group) => group.options)
+      : options || [];
+
+    return allOptions
       .filter((option) => !!option.key && selectedValueKeys.has(option.key))
       .map((option) => ({
         key: option.key as string,
@@ -77,9 +97,77 @@ export function MultiSelectKeyValues<
       })) as T[];
   }
 
+  const filterOptions = (options: MultiSelectOptions[]) => {
+    if (!searchQuery.trim()) return options;
+    const searchLower = searchQuery.toLowerCase().trim();
+
+    return options.filter((option) => {
+      const valueLower = option.value.toLowerCase();
+      const keyLower = option.key?.toLowerCase() || "";
+      return valueLower.includes(searchLower) || keyLower.includes(searchLower);
+    });
+  };
+
+  const renderOption = (option: MultiSelectOptions) => {
+    const isSelected = selectedValueKeys.has(option.key ?? option.value);
+    return (
+      <DropdownMenuCheckboxItem
+        key={option.key ?? option.value}
+        checked={isSelected}
+        onSelect={(e) => e.preventDefault()}
+        onCheckedChange={() => {
+          const value = option.key ?? option.value;
+          if (isSelected) {
+            selectedValueKeys.delete(value);
+          } else {
+            selectedValueKeys.add(value);
+          }
+          const filterValues = formatFilterValues();
+          onValueChange(
+            filterValues.length ? filterValues : [],
+            value,
+            selectedValueKeys,
+          );
+        }}
+        disabled={option.disabled}
+        className="group"
+      >
+        <span
+          className={cn(
+            "capitalize",
+            option.isArchived ? "text-foreground/50" : "",
+          )}
+        >
+          {option.value}
+        </span>
+        {option.isArchived && (
+          <Archive className="ml-2 h-4 w-4 text-foreground/50" />
+        )}
+        {option.count !== undefined && (
+          <span className="ml-auto font-mono text-xs">{option.count}</span>
+        )}
+      </DropdownMenuCheckboxItem>
+    );
+  };
+
+  const inputRef = React.useRef<HTMLInputElement>(null);
+
+  const handleInputClick = (e: React.MouseEvent) => {
+    e.stopPropagation();
+    inputRef.current?.focus();
+  };
+
   return (
-    <Popover modal>
-      <PopoverTrigger asChild>
+    <DropdownMenu
+      open={isOpen}
+      onOpenChange={(open) => {
+        setIsOpen(open);
+        if (!open) {
+          setSearchQuery("");
+        }
+      }}
+    >
+      <DropdownMenuTrigger asChild>
         <Button
           variant="outline"
           className={cn(
@@ -88,7 +176,9 @@ export function MultiSelectKeyValues<
           )}
           disabled={disabled}
         >
+          {iconLeft}
           {title}
+          {iconRight}
           <ChevronDown className="h-4 w-4 opacity-50" />
           {selectedValueKeys.size > 0 && (
             <>
@@ -126,91 +216,84 @@ export function MultiSelectKeyValues<
             </>
           )}
         </Button>
-      </PopoverTrigger>
-      <PopoverContent className="w-[200px] p-0" align={align}>
-        <Command>
-          <CommandInput placeholder={placeholder} />
-          <CommandList>
-            <CommandEmpty>No results found.</CommandEmpty>
-            <CommandGroup>
-              {Boolean(options.length) ? (
-                options.map((option) => {
-                  const isSelected = selectedValueKeys.has(
-                    option.key ?? option.value,
-                  );
-                  return (
-                    <CommandItem
-                      key={option.key ?? option.value}
-                      value={option.key ?? option.value}
-                      keywords={[option.value]}
-                      onSelect={(value) => {
-                        if (isSelected) {
-                          selectedValueKeys.delete(value);
-                        } else {
-                          selectedValueKeys.add(value);
-                        }
-                        const filterValues = formatFilterValues();
+      </DropdownMenuTrigger>
+      <DropdownMenuContent
+        align={align}
+        className="w-[200px]"
+        onPointerDownOutside={() => setIsOpen(false)}
+      >
+        <div
+          className="flex items-center border-b px-2 py-1"
+          onClick={handleInputClick}
+        >
+          <Search className="mr-1 h-3 w-3 opacity-50" />
+          <Input
+            ref={inputRef}
+            placeholder={placeholder}
+            value={searchQuery}
+            onChange={(e) => setSearchQuery(e.target.value)}
+            className="h-6 border-0 bg-transparent p-0 text-sm focus-visible:ring-0"
+            autoComplete="off"
+            autoCorrect="off"
+            spellCheck={false}
+            onKeyDown={(e) => e.stopPropagation()}
+            onClick={(e) => e.stopPropagation()}
+          />
+        </div>
+        <div className="max-h-[300px] overflow-y-auto">
+          {options &&
+            options.length > 0 &&
+            filterOptions(Array.from(options)).map(renderOption)}
 
-                        onValueChange(
-                          filterValues.length ? filterValues : [],
-                          value,
-                          selectedValueKeys,
-                        );
-                      }}
-                      disabled={option.disabled}
-                    >
-                      <div
-                        className={cn(
-                          "mr-2 flex h-4 w-4 items-center justify-center rounded-sm border border-primary",
-                          isSelected
-                            ? "bg-primary text-primary-foreground"
-                            : "opacity-50 [&_svg]:invisible",
-                          option.disabled ? "opacity-50" : null,
-                        )}
-                      >
-                        <Check className="h-4 w-4" />
-                      </div>
-                      <span
-                        className={cn(
-                          "overflow-x-scroll capitalize",
-                          option.isArchived ? "text-foreground/50" : "",
-                        )}
-                      >
-                        {option.value}
-                      </span>
-                      {option.isArchived ? (
-                        <div className="ml-1 mt-1 flex h-4 w-4">
-                          <Archive className="h-4 w-4 text-foreground/50"></Archive>
-                        </div>
-                      ) : null}
-                      {option.count !== undefined ? (
-                        <span className="ml-auto flex h-4 w-4 items-center justify-center pl-1 font-mono text-xs">
-                          {option.count}
-                        </span>
-                      ) : null}
-                    </CommandItem>
-                  );
-                })
-              ) : (
-                <CommandItem disabled>No options found.</CommandItem>
-              )}
-            </CommandGroup>
-            {controlButtons || showClearItems ? (
-              <>
-                <CommandSeparator />
-                <CommandGroup heading="Controls">
-                  {showClearItems && (
-                    <CommandItem onSelect={() => onValueChange([])}>
-                      Clear {items}
-                    </CommandItem>
-                  )}
-                  {controlButtons}
-                </CommandGroup>
-              </>
-            ) : null}
-          </CommandList>
-        </Command>
-      </PopoverContent>
-    </Popover>
+          {groupedOptions?.map((group) => {
+            const filteredGroupOptions = filterOptions(group.options);
+            if (filteredGroupOptions.length === 0) return null;
+
+            return (
+              <DropdownMenuSub key={group.label}>
+                <DropdownMenuSubTrigger className="flex w-full cursor-default select-none items-center">
+                  <Component className="mr-2 h-4 w-4 opacity-50" />
+                  <span>{group.label}</span>
+                </DropdownMenuSubTrigger>
+                <DropdownMenuSubContent className="max-h-[300px] overflow-y-auto">
+                  {filteredGroupOptions.map(renderOption)}
+                </DropdownMenuSubContent>
+              </DropdownMenuSub>
+            );
+          })}
+
+          {searchQuery &&
+            (!options || filterOptions(Array.from(options)).length === 0) &&
+            (!groupedOptions ||
+              !groupedOptions.some(
+                (group) => filterOptions(group.options).length > 0,
+              )) && (
+              <div className="px-2 py-1.5 text-sm text-muted-foreground">
+                No results found.
+              </div>
+            )}
+
+          {showClearItems && !searchQuery && (
+            <>
+              <DropdownMenuSeparator />
+              <DropdownMenuItem
+                onSelect={(e) => {
+                  e.preventDefault();
+                  onValueChange([]);
+                }}
+              >
+                Clear {items}
+              </DropdownMenuItem>
+            </>
+          )}
+          {controlButtons && (
+            <>
+              <DropdownMenuSeparator />
+              {controlButtons}
+            </>
+          )}
+        </div>
+      </DropdownMenuContent>
+    </DropdownMenu>
   );
 }

web/src/features/scores/types.ts

@@ -0,0 +1,30 @@
+import { type ScoreDataType } from "@langfuse/shared";
+
+export type HistogramBin = { binLabel: string; count: number };
+export type CategoryCounts = Record<string, number>;
+export type ChartBin = { binLabel: string } & CategoryCounts;
+
+export type TimeseriesChartProps = {
+  chartData: ChartBin[];
+  chartLabels: string[];
+  title: string;
+  type: "numeric" | "categorical";
+  index?: string;
+};
+
+export type ChartData = {
+  chartData: ChartBin[];
+  chartLabels: string[];
+};
+
+export type ScoreData = {
+  key: string;
+  name: string;
+  dataType: ScoreDataType;
+  source: string;
+};
+
+// Adapter interface to standardize data transformation
+export interface TimeseriesDataTransformer {
+  toChartData(): ChartData;
+}

web/src/features/setup/components/SetupTracingButton.tsx

@@ -6,7 +6,7 @@ import { setupTracingRoute } from "@/src/features/setup/setupRoutes";
 import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import { LockIcon } from "lucide-react";
 import { useRouter } from "next/router";
-import { useEffect } from "react";
+import { useEffect, useRef } from "react";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 const SetupTracingButton = () => {
@@ -27,10 +27,13 @@ const SetupTracingButton = () => {
     },
   );
 
+  // dedupe result via useRef, otherwise we'll capture the event multiple times on session refresh
+  const capturedEventAlready = useRef<boolean | undefined>(undefined);
   const capture = usePostHogClientCapture();
   useEffect(() => {
-    if (hasAnyTrace !== undefined) {
+    if (hasAnyTrace !== undefined && !capturedEventAlready.current) {
       capture("onboarding:tracing_check_active", { active: hasAnyTrace });
+      capturedEventAlready.current = true;
     }
   }, [hasAnyTrace, capture]);
 

web/src/pages/_app.tsx

@@ -12,7 +12,7 @@ import { QueryParamProvider } from "use-query-params";
 
 import "@/src/styles/globals.css";
 import Layout from "@/src/components/layouts/layout";
-import { useEffect } from "react";
+import { useEffect, useRef } from "react";
 import { useRouter } from "next/router";
 
 import posthog from "posthog-js";
@@ -62,6 +62,7 @@ if (
       if (process.env.NODE_ENV === "development") posthog.debug();
     },
     autocapture: false,
+    enable_heatmaps: false,
   });
 }
 
@@ -126,8 +127,16 @@ function UserTracking() {
   const sessionUser = session.data?.user;
   const { organization, project } = useQueryProjectOrOrganization();
 
+  // dedupe the event via useRef, otherwise we'll capture the event multiple times on session refresh
+  const lastIdentifiedUser = useRef<string | null>(null);
+
   useEffect(() => {
-    if (sessionUser) {
+    if (
+      session.status === "authenticated" &&
+      sessionUser &&
+      lastIdentifiedUser.current !== JSON.stringify(sessionUser)
+    ) {
+      lastIdentifiedUser.current = JSON.stringify(sessionUser);
       // PostHog
       if (env.NEXT_PUBLIC_POSTHOG_KEY && env.NEXT_PUBLIC_POSTHOG_HOST)
         posthog.identify(sessionUser.id ?? undefined, {
@@ -171,7 +180,8 @@ function UserTracking() {
             : "undefined",
         },
       });
-    } else {
+    } else if (session.status === "unauthenticated") {
+      lastIdentifiedUser.current = null;
       // PostHog
       if (env.NEXT_PUBLIC_POSTHOG_KEY && env.NEXT_PUBLIC_POSTHOG_HOST) {
         posthog.reset();
@@ -180,7 +190,7 @@ function UserTracking() {
       // Sentry
       setUser(null);
     }
-  }, [sessionUser]);
+  }, [sessionUser, session.status]);
 
   // update crisp segments
   const plan = organization?.plan;
@@ -193,7 +203,6 @@ function UserTracking() {
   useEffect(() => {
     let segments = [];
     if (plan && !currentOrgIsDemoOrg) {
-      console.log("setting chat segments", plan);
       segments.push("plan:" + plan);
     }
     if (currentOrgIsDemoOrg) {

web/src/pages/project/[projectId]/datasets/[datasetId]/compare.tsx

@@ -21,6 +21,20 @@ import {
 } from "@/src/components/ui/dialog";
 import { CreateExperimentsForm } from "@/src/ee/features/experiments/components/CreateExperimentsForm";
 import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { DatasetAnalytics } from "@/src/features/datasets/components/DatasetAnalytics";
+import { Card, CardContent } from "@/src/components/ui/card";
+import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
+import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
+import { TimeseriesChart } from "@/src/features/scores/components/TimeseriesChart";
+import {
+  isNumericDataType,
+  toOrderedScoresList,
+} from "@/src/features/scores/lib/helpers";
+import { CompareViewAdapter } from "@/src/features/scores/adapters";
+import {
+  RESOURCE_METRICS,
+  transformAggregatedRunMetricsToChartData,
+} from "@/src/features/dashboard/lib/score-analytics-utils";
 
 export default function DatasetCompare() {
   const router = useRouter();
@@ -29,11 +43,15 @@ export default function DatasetCompare() {
   const [runState, setRunState] = useQueryParams({
     runs: withDefault(ArrayParam, []),
   });
+
   const [isCreateExperimentDialogOpen, setIsCreateExperimentDialogOpen] =
     useState(false);
   const [localRuns, setLocalRuns] = useState<
     Array<{ key: string; value: string }>
   >([]);
+  const [selectedMetrics, setSelectedMetrics] = useState<string[]>(
+    RESOURCE_METRICS.map((metric) => metric.key),
+  );
   const runIds = runState.runs as undefined | string[];
 
   const hasExperimentWriteAccess = useHasProjectAccess({
@@ -59,6 +77,61 @@ export default function DatasetCompare() {
   );
   const utils = api.useUtils();
 
+  const runMetrics = api.datasets.runsByDatasetIdMetrics.useQuery(
+    {
+      projectId,
+      datasetId,
+      queryClickhouse: useClickhouse(),
+      runIds: runIds,
+    },
+    {
+      enabled: runIds && runIds.length > 1,
+    },
+  );
+
+  // LFE-3236: refactor to filter query to only include scores for runs in runIds
+  const scoreKeysAndProps = api.scores.getScoreKeysAndProps.useQuery(
+    {
+      projectId: projectId,
+      selectedTimeOption: { option: "All time", filterSource: "TABLE" },
+      queryClickhouse: useClickhouse(),
+    },
+    {
+      enabled: runIds && runIds.length > 1,
+    },
+  );
+
+  const scoreIdToName = useMemo(() => {
+    return new Map(
+      scoreKeysAndProps.data?.map((obj) => [obj.key, obj.name]) ?? [],
+    );
+  }, [scoreKeysAndProps.data]);
+
+  const runAggregatedMetrics = useMemo(() => {
+    return transformAggregatedRunMetricsToChartData(
+      runMetrics.data?.runs.filter((run) => runIds?.includes(run.id)) ?? [],
+      scoreIdToName,
+    );
+  }, [runMetrics.data, runIds, scoreIdToName]);
+
+  const { scoreAnalyticsOptions, scoreKeyToData } = useMemo(() => {
+    const scoreAnalyticsOptions = scoreKeysAndProps.data
+      ? toOrderedScoresList(scoreKeysAndProps.data).map(
+          ({ key, name, dataType, source }) => ({
+            key,
+            value: `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`,
+          }),
+        )
+      : [];
+
+    return {
+      scoreAnalyticsOptions,
+      scoreKeyToData: new Map(
+        scoreKeysAndProps.data?.map((obj) => [obj.key, obj]) ?? [],
+      ),
+    };
+  }, [scoreKeysAndProps.data]);
+
   const handleExperimentSettled = async (data?: {
     success: boolean;
     datasetId: string;
@@ -89,7 +162,7 @@ export default function DatasetCompare() {
   }
 
   return (
-    <FullScreenPage key={runIds?.join(",") ?? "empty"}>
+    <FullScreenPage>
       <Header
         title={`Compare runs: ${dataset.data?.name ?? datasetId}`}
         breadcrumb={[
@@ -154,6 +227,15 @@ export default function DatasetCompare() {
               </div>
             </PopoverContent>
           </Popover>,
+          runIds && runIds.length > 1 ? (
+            <DatasetAnalytics
+              key="dataset-analytics"
+              projectId={projectId}
+              scoreOptions={scoreAnalyticsOptions}
+              selectedMetrics={selectedMetrics}
+              setSelectedMetrics={setSelectedMetrics}
+            />
+          ) : null,
           <MultiSelectKeyValues
             key="select-runs"
             title="Select runs"
@@ -185,6 +267,52 @@ export default function DatasetCompare() {
           />,
         ]}
       />
+      {Boolean(selectedMetrics.length) &&
+        Boolean(runAggregatedMetrics?.size) && (
+          <Card className="my-4 max-h-[25dvh] md:max-h-[30dvh]">
+            <CardContent className="mt-2 h-full">
+              <div className="flex h-full w-full gap-4 overflow-x-auto">
+                {selectedMetrics.map((key) => {
+                  const adapter = new CompareViewAdapter(
+                    runAggregatedMetrics,
+                    key,
+                  );
+                  const { chartData, chartLabels } = adapter.toChartData();
+
+                  const scoreData = scoreKeyToData.get(key);
+                  if (!scoreData)
+                    return (
+                      <TimeseriesChart
+                        key={key}
+                        chartData={chartData}
+                        chartLabels={chartLabels}
+                        title={
+                          RESOURCE_METRICS.find((metric) => metric.key === key)
+                            ?.label ?? key
+                        }
+                        type="numeric"
+                      />
+                    );
+
+                  return (
+                    <TimeseriesChart
+                      key={key}
+                      chartData={chartData}
+                      chartLabels={chartLabels}
+                      title={`${getScoreDataTypeIcon(scoreData.dataType)} ${scoreData.name} (${scoreData.source.toLowerCase()})`}
+                      type={
+                        isNumericDataType(scoreData.dataType)
+                          ? "numeric"
+                          : "categorical"
+                      }
+                    />
+                  );
+                })}
+              </div>
+            </CardContent>
+          </Card>
+        )}
+
       <DatasetCompareRunsTable
         key={runIds?.join(",") ?? "empty"}
         projectId={projectId}

web/src/pages/project/[projectId]/datasets/[datasetId]/index.tsx

@@ -7,13 +7,11 @@ import Link from "next/link";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { DatasetActionButton } from "@/src/features/datasets/components/DatasetActionButton";
 import { DeleteButton } from "@/src/components/deleteButton";
-import { JSONView } from "@/src/components/ui/CodeJsonViewer";
 import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
 import { DuplicateDatasetButton } from "@/src/features/datasets/components/DuplicateDatasetButton";
 import { useState } from "react";
 import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";
-import { CommandItem } from "@/src/components/ui/command";
-import { ExternalLink, FlaskConical } from "lucide-react";
+import { ExternalLink, FlaskConical, FolderKanban } from "lucide-react";
 import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import { useMemo } from "react";
 import { useHasEntitlement } from "@/src/features/entitlements/hooks";
@@ -25,6 +23,15 @@ import {
 import { Button } from "@/src/components/ui/button";
 import { CreateExperimentsForm } from "@/src/ee/features/experiments/components/CreateExperimentsForm";
 import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+import { DropdownMenuItem } from "@/src/components/ui/dropdown-menu";
+import { DatasetAnalytics } from "@/src/features/datasets/components/DatasetAnalytics";
+import { RESOURCE_METRICS } from "@/src/features/dashboard/lib/score-analytics-utils";
+import { MarkdownOrJsonView } from "@/src/components/trace/IOPreview";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
 
 export default function Dataset() {
   const router = useRouter();
@@ -34,6 +41,15 @@ export default function Dataset() {
   const hasEntitlement = useHasEntitlement("model-based-evaluations");
   const [isCreateExperimentDialogOpen, setIsCreateExperimentDialogOpen] =
     useState(false);
+  const [selectedMetrics, setSelectedMetrics] = useState<string[]>(
+    RESOURCE_METRICS.map((metric) => metric.key),
+  );
+  const [scoreOptions, setScoreOptions] = useState<
+    {
+      key: string;
+      value: string;
+    }[]
+  >([]);
 
   const dataset = api.datasets.byId.useQuery({
     datasetId,
@@ -132,6 +148,14 @@ export default function Dataset() {
               </DialogContent>
             </Dialog>
 
+            <DatasetAnalytics
+              key="dataset-analytics"
+              projectId={projectId}
+              scoreOptions={scoreOptions}
+              selectedMetrics={selectedMetrics}
+              setSelectedMetrics={setSelectedMetrics}
+            />
+
             {hasReadAccess && hasEntitlement && evaluators.isSuccess && (
               <MultiSelectKeyValues
                 className="max-w-fit"
@@ -148,17 +172,41 @@ export default function Dataset() {
                 values={evaluatorsOptions}
                 options={evaluatorsOptions}
                 controlButtons={
-                  <CommandItem
+                  <DropdownMenuItem
                     onSelect={() => {
                       window.open(`/project/${projectId}/evals`, "_blank");
                     }}
                   >
                     Manage evaluators
                     <ExternalLink className="ml-auto h-4 w-4" />
-                  </CommandItem>
+                  </DropdownMenuItem>
                 }
               />
             )}
+            <Popover key="show-dataset-details">
+              <PopoverTrigger asChild>
+                <Button variant="outline">
+                  <FolderKanban className="mr-2 h-4 w-4" />
+                  Dataset details
+                </Button>
+              </PopoverTrigger>
+              <PopoverContent className="mx-2 max-h-[50vh] w-[50vw] overflow-y-auto md:w-[25vw]">
+                <div className="space-y-4">
+                  <div>
+                    <h4 className="mb-1 font-medium">Description</h4>
+                    <span className="text-sm text-muted-foreground">
+                      {dataset.data?.description ?? "No description"}
+                    </span>
+                  </div>
+                  <div>
+                    <h4 className="mb-1 font-medium">Metadata</h4>
+                    <MarkdownOrJsonView
+                      content={dataset.data?.metadata ?? null}
+                    />
+                  </div>
+                </div>
+              </PopoverContent>
+            </Popover>
             <DetailPageNav
               currentId={datasetId}
               path={(entry) => `/project/${projectId}/datasets/${entry.id}`}
@@ -190,17 +238,12 @@ export default function Dataset() {
           </>
         }
       />
-      {!!dataset.data?.metadata && (
-        <JSONView
-          json={dataset?.data.metadata}
-          title="Metadata"
-          className="max-h-[25vh] overflow-y-auto"
-        />
-      )}
 
       <DatasetRunsTable
         projectId={projectId}
         datasetId={datasetId}
+        selectedMetrics={selectedMetrics}
+        setScoreOptions={setScoreOptions}
         menuItems={
           <Tabs value="runs">
             <TabsList>

web/src/pages/project/[projectId]/models.tsx

@@ -1,13 +1,12 @@
-import Header from "@/src/components/layouts/header";
-
 import { useRouter } from "next/router";
+
+import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
+import Header from "@/src/components/layouts/header";
 import ModelTable from "@/src/components/table/use-cases/models";
 import { Button } from "@/src/components/ui/button";
-import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
-import { Lock } from "lucide-react";
-import Link from "next/link";
+import { UpsertModelFormDrawer } from "@/src/features/models/components/UpsertModelFormDrawer";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
-import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 
 export default function ModelsPage() {
   const router = useRouter();
@@ -27,19 +26,15 @@ export default function ModelsPage() {
           href: "https://langfuse.com/docs/model-usage-and-cost",
         }}
         actionButtons={
-          <Button
-            variant="secondary"
-            disabled={!hasWriteAccess}
-            onClick={() => capture("models:new_form_open")}
-            asChild
-          >
-            <Link
-              href={hasWriteAccess ? `/project/${projectId}/models/new` : "#"}
+          <UpsertModelFormDrawer {...{ projectId, action: "create" }}>
+            <Button
+              variant="secondary"
+              disabled={!hasWriteAccess}
+              onClick={() => capture("models:new_form_open")}
             >
-              {!hasWriteAccess && <Lock size={16} className="mr-2" />}
               Add model definition
-            </Link>
-          </Button>
+            </Button>
+          </UpsertModelFormDrawer>
         }
       />
       <ModelTable projectId={projectId} />

web/src/pages/project/[projectId]/models/[modelId].tsx

@@ -0,0 +1,208 @@
+import { useRouter } from "next/router";
+import { api } from "@/src/utils/api";
+import { FullScreenPage } from "@/src/components/layouts/full-screen-page";
+import Header from "@/src/components/layouts/header";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import {
+  Card,
+  CardContent,
+  CardHeader,
+  CardTitle,
+} from "@/src/components/ui/card";
+import { DeleteModelButton } from "@/src/features/models/components/DeleteModelButton";
+import { EditModelButton } from "@/src/features/models/components/EditModelButton";
+import { CloneModelButton } from "@/src/features/models/components/CloneModelButton";
+import { JSONView } from "@/src/components/ui/CodeJsonViewer";
+import Link from "next/link";
+import { Button } from "@/src/components/ui/button";
+import { getMaxDecimals } from "@/src/features/models/utils";
+import Decimal from "decimal.js";
+import { PriceUnitSelector } from "@/src/features/models/components/PriceUnitSelector";
+import { useMemo } from "react";
+import { usePriceUnitMultiplier } from "@/src/features/models/hooks/usePriceUnitMultiplier";
+import Generations from "@/src/components/table/use-cases/generations";
+import { ArrowTopRightIcon } from "@radix-ui/react-icons";
+
+export default function ModelDetailPage() {
+  const router = useRouter();
+  const { priceUnit, priceUnitMultiplier } = usePriceUnitMultiplier();
+  const projectId = router.query.projectId as string;
+  const modelId = router.query.modelId as string;
+  const hasWriteAccess = useHasProjectAccess({
+    projectId,
+    scope: "models:CUD",
+  });
+
+  const { data: model, isLoading } = api.models.getById.useQuery(
+    { projectId, modelId },
+    { enabled: !!projectId && !!modelId },
+  );
+
+  const maxDecimals = useMemo(
+    () =>
+      Math.max(
+        ...Object.values(model?.prices ?? {}).map((price) =>
+          getMaxDecimals(price, priceUnitMultiplier),
+        ),
+      ),
+    [model?.prices, priceUnitMultiplier],
+  );
+
+  // If not found, redirect to models page
+  if (!isLoading && !model) {
+    return (
+      <div className="flex min-h-screen flex-col items-center justify-center">
+        <div className="mb-4 text-xl font-medium">Model not found</div>
+        <Button variant="outline" asChild>
+          <Link href={`/project/${projectId}/models`}>
+            Return to Models page
+          </Link>
+        </Button>
+      </div>
+    );
+  }
+
+  const isLangfuseModel = !Boolean(model?.projectId);
+
+  if (isLoading || !model) {
+    return <div>Loading...</div>;
+  }
+
+  return (
+    <FullScreenPage>
+      <Header
+        title={model.modelName}
+        help={{
+          description: "Model configuration and pricing details",
+          href: "https://langfuse.com/docs/model-usage-and-cost",
+        }}
+        breadcrumb={[
+          {
+            name: "Models",
+            href: `/project/${router.query.projectId as string}/models`,
+          },
+          { name: model.modelName },
+        ]}
+        actionButtons={
+          <div className="flex gap-2">
+            {hasWriteAccess &&
+              (!isLangfuseModel ? (
+                <>
+                  <EditModelButton projectId={projectId} modelData={model} />
+                  <DeleteModelButton
+                    projectId={projectId}
+                    modelData={model}
+                    onSuccess={() => {
+                      void router.push(`/project/${projectId}/models`);
+                    }}
+                  />
+                </>
+              ) : (
+                <CloneModelButton projectId={projectId} modelData={model} />
+              ))}
+          </div>
+        }
+      />
+
+      <div className="grid grid-cols-2 gap-6 p-2">
+        <Card>
+          <CardHeader>
+            <CardTitle>Model configuration</CardTitle>
+          </CardHeader>
+          <CardContent className="grid gap-4">
+            <div>
+              <div className="text-sm font-medium text-muted-foreground">
+                Match Pattern
+              </div>
+              <div className="mt-1 font-mono text-sm">{model.matchPattern}</div>
+            </div>
+
+            <div>
+              <div className="text-sm font-medium text-muted-foreground">
+                Maintained by
+              </div>
+              <div className="mt-1 text-sm">
+                {isLangfuseModel ? "Langfuse" : "User"}
+              </div>
+            </div>
+
+            <div>
+              <div className="text-sm font-medium text-muted-foreground">
+                Tokenizer
+              </div>
+              <div className="mt-1 text-sm">{model.tokenizerId || "None"}</div>
+            </div>
+
+            {model.tokenizerId && (
+              <div>
+                <div className="text-sm font-medium text-muted-foreground">
+                  Tokenizer Config
+                </div>
+                <pre className="mt-1 rounded bg-muted p-2 text-sm">
+                  <JSONView json={model.tokenizerConfig} />
+                </pre>
+              </div>
+            )}
+          </CardContent>
+        </Card>
+
+        <Card>
+          <CardHeader>
+            <CardTitle>Pricing</CardTitle>
+          </CardHeader>
+          <CardContent>
+            <div className="flex flex-col gap-2">
+              <div className="grid grid-cols-2 gap-2 border-b border-border text-sm font-medium text-muted-foreground">
+                <span>Usage Type</span>
+                <span className="flex items-center gap-2">
+                  <span>Price {priceUnit}</span>
+                  <PriceUnitSelector />
+                </span>
+              </div>
+              {Object.entries(model.prices).map(([usageType, price]) => (
+                <div
+                  key={usageType}
+                  className="grid grid-cols-2 gap-2 rounded px-1 py-0.5 text-sm"
+                >
+                  <span className="break-all">{usageType}</span>
+                  <span className="text-left font-mono">
+                    $
+                    {new Decimal(price)
+                      .mul(priceUnitMultiplier)
+                      .toFixed(maxDecimals)}
+                  </span>
+                </div>
+              ))}
+            </div>
+          </CardContent>
+        </Card>
+
+        <Card className="col-span-2">
+          <CardHeader>
+            <CardTitle className="flex items-center justify-between">
+              <span>Model generations</span>
+              <Button variant="ghost" asChild>
+                <Link
+                  href={`/project/${projectId}/generations`}
+                  className="flex items-center gap-1"
+                >
+                  <span className="text-sm">View all</span>
+                  <ArrowTopRightIcon className="h-4 w-4" />
+                </Link>
+              </Button>
+            </CardTitle>
+          </CardHeader>
+          <CardContent>
+            <div className="flex max-h-[calc(100vh-20rem)] flex-col">
+              <Generations
+                projectId={projectId}
+                omittedFilter={["Model"]}
+                modelId={model.id}
+              />
+            </div>
+          </CardContent>
+        </Card>
+      </div>
+    </FullScreenPage>
+  );
+}

web/src/pages/project/[projectId]/models/new.tsx

@@ -1,36 +0,0 @@
-import Header from "@/src/components/layouts/header";
-
-import { useRouter } from "next/router";
-import { NewModelForm } from "@/src/features/models/components/NewModelForm";
-import { ScrollScreenPage } from "@/src/components/layouts/scroll-screen-page";
-
-export default function ModelsPage() {
-  const router = useRouter();
-  const projectId = router.query.projectId as string;
-
-  return (
-    <ScrollScreenPage>
-      <Header
-        title="New Model Definition"
-        breadcrumb={[
-          {
-            name: "Models",
-            href: `/project/${projectId}/models`,
-          },
-          {
-            name: "New",
-          },
-        ]}
-        help={{
-          description:
-            "Create a project-specific model definition. This will be used by Langfuse to infer model usage (eg tokens) and cost (USD).",
-          href: "https://langfuse.com/docs/model-usage-and-cost",
-        }}
-      />
-      <NewModelForm
-        projectId={projectId}
-        onFormSuccess={() => void router.push(`/project/${projectId}/models`)}
-      />
-    </ScrollScreenPage>
-  );
-}

web/src/pages/project/[projectId]/users.tsx

@@ -89,6 +89,7 @@ export default function UsersPage() {
       projectId,
       userIds: users.data?.users.map((u) => u.userId) ?? [],
       queryClickhouse: useClickhouse(),
+      filter: filterState,
     },
     {
       enabled: users.isSuccess,

web/src/server/api/routers/generations/filterOptionsQuery.ts

@@ -11,6 +11,7 @@ import {
   datetimeFilterToPrisma,
   datetimeFilterToPrismaSql,
   getObservationsGroupedByModel,
+  getObservationsGroupedByModelId,
   getObservationsGroupedByName,
   getObservationsGroupedByPromptName,
   getScoresGroupedByName,
@@ -87,7 +88,7 @@ export const filterOptionsQuery = protectedProjectProcedure
     };
 
     // Score names
-    const [scores, model, name, promptNames, traceNames, tags] =
+    const [scores, model, name, promptNames, traceNames, tags, modelId] =
       await measureAndReturnApi({
         input,
         operation: "traces.all",
@@ -175,6 +176,8 @@ export const filterOptionsQuery = protectedProjectProcedure
             ${rawStartTimeFilter}
           LIMIT 1000;
       `),
+            // modelId
+            [] as any[],
           ]);
         },
         clickhouseExecution: async () => {
@@ -212,6 +215,11 @@ export const filterOptionsQuery = protectedProjectProcedure
             getClickhouseTraceName(),
             // trace tags
             getClickhouseTraceTags(),
+            // modelId
+            getObservationsGroupedByModelId(
+              input.projectId,
+              startTimeFilter ? [startTimeFilter] : [],
+            ),
           ]);
         },
       });
@@ -221,6 +229,11 @@ export const filterOptionsQuery = protectedProjectProcedure
       model: model
         .filter((i) => i.model !== null)
         .map((i) => ({ value: i.model as string })),
+      modelId: modelId
+        .filter((i) => i.modelId !== null)
+        .map((i) => ({
+          value: i.modelId as string,
+        })),
       name: name
         .filter((i) => i.name !== null)
         .map((i) => ({ value: i.name as string })),

web/src/server/api/routers/models.ts

@@ -1,15 +1,19 @@
+import { v4 as uuidv4 } from "uuid";
 import { z } from "zod";
 
-import { ModelUsageUnit } from "@langfuse/shared";
+import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { isValidPostgresRegex } from "@/src/features/models/server/isValidPostgresRegex";
+import {
+  GetModelResultSchema,
+  UpsertModelSchema,
+} from "@/src/features/models/validation";
 import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 import {
   createTRPCRouter,
   protectedProjectProcedure,
 } from "@/src/server/api/trpc";
-import { paginationZod } from "@langfuse/shared";
+import { ModelUsageUnit, paginationZod } from "@langfuse/shared";
 import { TRPCError } from "@trpc/server";
-import { auditLog } from "@/src/features/audit-logs/auditLog";
-import { isValidPostgresRegex } from "@/src/features/models/server/isValidPostgresRegex";
 
 const ModelAllOptions = z.object({
   projectId: z.string(),
@@ -17,53 +21,228 @@ const ModelAllOptions = z.object({
 });
 
 export const modelRouter = createTRPCRouter({
-  all: protectedProjectProcedure
+  getById: protectedProjectProcedure
+    .input(z.object({ projectId: z.string(), modelId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      const modelQueryResult = await ctx.prisma.$queryRaw`          
+          SELECT 
+            m.id,
+            m.project_id as "projectId",
+            m.model_name as "modelName",
+            m.match_pattern as "matchPattern",
+            m.tokenizer_config as "tokenizerConfig",
+            m.tokenizer_id as "tokenizerId",
+            COALESCE(
+              (
+                SELECT
+                  JSONB_OBJECT_AGG(usage_type, price)
+                FROM
+                  prices
+                WHERE
+                  model_id = m.id
+              ),
+              '{}'::jsonb
+            ) AS prices
+          FROM
+            models m
+          WHERE
+            m.id = ${input.modelId}
+            AND (
+              project_id IS NULL
+              OR project_id = ${input.projectId}
+            );
+      `;
+
+      const model = z.array(GetModelResultSchema).parse(modelQueryResult)[0];
+
+      if (!model || (model.projectId && model.projectId !== input.projectId)) {
+        throw new TRPCError({
+          code: "NOT_FOUND",
+          message: "Model not found",
+        });
+      }
+
+      return model;
+    }),
+
+  getAll: protectedProjectProcedure
     .input(ModelAllOptions)
     .query(async ({ input, ctx }) => {
-      const [models, totalAmount] = await Promise.all([
-        ctx.prisma.model.findMany({
-          where: {
-            OR: [{ projectId: input.projectId }, { projectId: null }],
-          },
-          skip: input.page * input.limit,
-          orderBy: [
-            { modelName: "asc" },
-            { unit: "asc" },
-            {
-              startDate: {
-                sort: "desc",
-                nulls: "last",
-              },
-            },
-          ],
-          take: input.limit,
-        }),
-        ctx.prisma.model.count({
-          where: {
-            OR: [{ projectId: input.projectId }, { projectId: null }],
-          },
-        }),
+      const [allModelsQueryResult, totalCountQuery] = await Promise.all([
+        // All models
+        ctx.prisma.$queryRaw`
+          SELECT DISTINCT ON (project_id, model_name) 
+            m.id,
+            m.project_id as "projectId",
+            m.model_name as "modelName",
+            m.match_pattern as "matchPattern",
+            m.tokenizer_config as "tokenizerConfig",
+            m.tokenizer_id as "tokenizerId",
+            COALESCE(
+              (
+                SELECT
+                  JSONB_OBJECT_AGG(usage_type, price)
+                FROM
+                  prices
+                WHERE
+                  model_id = m.id
+              ),
+              '{}'::jsonb
+            ) AS prices
+          FROM
+            models m
+          WHERE
+            project_id IS NULL
+            OR project_id = ${input.projectId}
+          ORDER BY
+            project_id,
+            model_name,
+            m.created_at DESC NULLS LAST 
+          LIMIT ${input.limit} OFFSET ${input.page * input.limit};
+          `,
+
+        // Total count
+        ctx.prisma.$queryRaw<
+          {
+            count: number;
+          }[]
+        >`
+          SELECT COUNT(DISTINCT (project_id, model_name))
+          FROM models
+          WHERE project_id IS NULL 
+          OR project_id = '${input.projectId}';
+        `,
       ]);
+
+      const allModels = z
+        .array(GetModelResultSchema)
+        .parse(allModelsQueryResult);
+      const totalCount = z.coerce.number().parse(totalCountQuery[0].count);
+
       return {
-        models,
-        totalCount: totalAmount,
+        models: allModels,
+        totalCount,
       };
     }),
-  modelNames: protectedProjectProcedure
-    .input(z.object({ projectId: z.string() }))
-    .query(async ({ input, ctx }) => {
-      return (
-        await ctx.prisma.model.findMany({
-          select: {
-            modelName: true,
-          },
-          distinct: ["modelName"],
-          orderBy: [{ modelName: "asc" }],
+  upsert: protectedProjectProcedure
+    .input(UpsertModelSchema)
+    .mutation(async ({ input, ctx }) => {
+      const {
+        modelId: providedModelId,
+        projectId,
+        modelName,
+        matchPattern,
+        tokenizerConfig,
+        tokenizerId,
+      } = input;
+
+      throwIfNoProjectAccess({
+        session: ctx.session,
+        projectId,
+        scope: "models:CUD",
+      });
+
+      // Check if regex is valid POSIX regex
+      // Use DB to check, because JS regex is not POSIX compliant
+      const isValidRegex = await isValidPostgresRegex(
+        input.matchPattern,
+        ctx.prisma,
+      );
+      if (!isValidRegex) {
+        throw new TRPCError({
+          code: "BAD_REQUEST",
+          message: "Invalid regex, needs to be Postgres syntax",
+        });
+      }
+
+      const modelId = providedModelId ?? uuidv4();
+
+      return await ctx.prisma.$transaction(async (tx) => {
+        // Check whether model belongs to project
+        // This check is important to prevent users from updating prices for models that they do not have access to
+        const existingModel = await tx.model.findUnique({
           where: {
-            OR: [{ projectId: input.projectId }, { projectId: null }],
+            id: modelId,
           },
-        })
-      ).map((model) => model.modelName);
+        });
+
+        if (existingModel && existingModel.projectId !== projectId) {
+          throw new TRPCError({
+            code: "NOT_FOUND",
+            message: "Model not found",
+          });
+        }
+
+        // Check if model name is unique within the project
+        // Note: The database has a uniqueness constraint on (projectId, modelName, startDate, unit),
+        // but this constraint is not enforced when startDate or unit are NULL.
+        // We do an explicit check here to ensure uniqueness on just (projectId, modelName).
+        // TODO(LFE-3229): After models table cleanup, enforce uniqueness constraint directly on (projectId, modelName)
+        const existingModelName = await tx.model.findFirst({
+          where: {
+            projectId,
+            modelName,
+          },
+        });
+
+        if (existingModelName && modelId !== existingModelName.id) {
+          throw new TRPCError({
+            code: "BAD_REQUEST",
+            message: `Model name '${modelName}' already exists in project`,
+          });
+        }
+
+        const upsertedModel = await tx.model.upsert({
+          where: {
+            id: modelId,
+            projectId: projectId,
+          },
+          create: {
+            id: modelId,
+            projectId,
+            modelName,
+            matchPattern,
+            tokenizerConfig,
+            tokenizerId,
+            startDate: new Date("2010-01-01"), // Set fix start date for uniqueness constraint to work. TODO: drop after cleanup of models table in LFE-3229
+            unit: ModelUsageUnit.Tokens, // Set fix unit for uniqueness constraint to work. TODO: drop after cleanup of models table in LFE-3229
+          },
+          update: {
+            matchPattern,
+            tokenizerConfig,
+            tokenizerId,
+          },
+        });
+
+        await tx.price.deleteMany({
+          where: {
+            modelId: upsertedModel.id,
+          },
+        });
+
+        await tx.price.createMany({
+          data: Object.entries(input.prices)
+            .filter(
+              (priceEntry): priceEntry is [string, number] =>
+                priceEntry[1] != null,
+            )
+            .map(([usageType, price]) => ({
+              modelId: upsertedModel.id,
+              usageType,
+              price,
+            })),
+        });
+
+        await auditLog({
+          session: ctx.session,
+          resourceType: "model",
+          resourceId: upsertedModel.id,
+          action: modelId ? "update" : "create",
+          after: upsertedModel,
+        });
+
+        return upsertedModel;
+      });
     }),
   delete: protectedProjectProcedure
     .input(
@@ -96,93 +275,4 @@ export const modelRouter = createTRPCRouter({
 
       return deletedModel;
     }),
-  create: protectedProjectProcedure
-    .input(
-      z.object({
-        projectId: z.string(),
-        modelName: z.string(),
-        matchPattern: z.string(),
-        startDate: z.date().optional(),
-        inputPrice: z.number().nonnegative().optional(),
-        outputPrice: z.number().nonnegative().optional(),
-        totalPrice: z.number().nonnegative().optional(),
-        unit: z.nativeEnum(ModelUsageUnit),
-        tokenizerId: z.enum(["openai", "claude"]).optional(),
-        tokenizerConfig: z.record(z.union([z.string(), z.number()])).optional(),
-      }),
-    )
-    .mutation(async ({ input, ctx }) => {
-      throwIfNoProjectAccess({
-        session: ctx.session,
-        projectId: input.projectId,
-        scope: "models:CUD",
-      });
-
-      // Check if regex is valid POSIX regex
-      // Use DB to check, because JS regex is not POSIX compliant
-
-      const isValidRegex = await isValidPostgresRegex(
-        input.matchPattern,
-        ctx.prisma,
-      );
-      if (!isValidRegex) {
-        throw new TRPCError({
-          code: "BAD_REQUEST",
-          message: "Invalid regex, needs to be Postgres syntax",
-        });
-      }
-
-      const createdModel = await ctx.prisma.$transaction(async (tx) => {
-        const createdModel = await tx.model.create({
-          data: {
-            projectId: input.projectId,
-            modelName: input.modelName,
-            matchPattern: input.matchPattern,
-            startDate: input.startDate,
-            inputPrice: input.inputPrice,
-            outputPrice: input.outputPrice,
-            totalPrice: input.totalPrice,
-            unit: input.unit,
-            tokenizerId: input.tokenizerId,
-            tokenizerConfig: input.tokenizerConfig,
-          },
-        });
-
-        // Populate prices table
-        const prices = [
-          { usageType: "input", price: input.inputPrice },
-          { usageType: "output", price: input.outputPrice },
-          { usageType: "total", price: input.totalPrice },
-        ];
-
-        const pricesToCreate = [];
-        for (const { usageType, price } of prices) {
-          if (price != null) {
-            pricesToCreate.push(
-              tx.price.create({
-                data: {
-                  modelId: createdModel.id,
-                  usageType,
-                  price,
-                },
-              }),
-            );
-          }
-        }
-
-        await Promise.all(pricesToCreate);
-
-        return createdModel;
-      });
-
-      await auditLog({
-        session: ctx.session,
-        resourceType: "model",
-        resourceId: createdModel.id,
-        action: "create",
-        after: createdModel,
-      });
-
-      return createdModel;
-    }),
 });

web/src/server/api/routers/traces.ts

@@ -64,13 +64,7 @@ type TraceFilterOptions = z.infer<typeof TraceFilterOptions>;
 
 export type ObservationReturnType = Omit<
   ObservationView,
-  | "input"
-  | "output"
-  | "modelId"
-  | "inputPrice"
-  | "outputPrice"
-  | "totalPrice"
-  | "metadata"
+  "input" | "output" | "inputPrice" | "outputPrice" | "totalPrice" | "metadata"
 > & {
   traceId: string;
   usageDetails: Record<string, number>;

web/src/server/api/routers/users.ts

@@ -121,6 +121,7 @@ export const userRouter = createTRPCRouter({
         projectId: z.string(),
         userIds: z.array(z.string().min(1)),
         queryClickhouse: z.boolean().default(false),
+        filter: z.array(singleFilter).nullable(),
       }),
     )
     .query(async ({ input, ctx }) => {
@@ -182,7 +183,11 @@ export const userRouter = createTRPCRouter({
           if (input.userIds.length === 0) {
             return [];
           }
-          const metrics = await getUserMetrics(input.projectId, input.userIds);
+          const metrics = await getUserMetrics(
+            input.projectId,
+            input.userIds,
+            input.filter ?? [],
+          );
 
           return metrics.map((metric) => ({
             userId: metric.userId,
@@ -262,7 +267,7 @@ export const userRouter = createTRPCRouter({
         },
         clickhouseExecution: async () => {
           const result = (
-            await getUserMetrics(input.projectId, [input.userId])
+            await getUserMetrics(input.projectId, [input.userId], [])
           ).shift();
 
           return {

web/src/server/auth.ts

@@ -189,6 +189,10 @@ if (
       authorization: {
         params: { scope: env.AUTH_CUSTOM_SCOPE ?? "openid email profile" },
       },
+      client: {
+        token_endpoint_auth_method:
+          env.AUTH_CUSTOM_CLIENT_AUTH_METHOD ?? "client_secret_basic",
+      },
     }),
   );
 

worker/package.json

@@ -1,6 +1,6 @@
 {
   "name": "worker",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "description": "",
   "license": "MIT",
   "private": true,

worker/src/__tests__/batch-export.test.ts

@@ -353,16 +353,22 @@ describe("batch export test suite", () => {
         project_id: projectId,
         trace_id: traces[0].id,
         type: "GENERATION",
+        start_time: new Date().getTime() - 1000,
+        end_time: new Date().getTime(),
       }),
       createObservation({
         project_id: projectId,
         trace_id: traces[1].id,
         type: "GENERATION",
+        start_time: new Date().getTime() - 2000,
+        end_time: new Date().getTime(),
       }),
       createObservation({
         project_id: projectId,
         trace_id: traces[1].id,
         type: "GENERATION",
+        start_time: new Date().getTime() - 2123,
+        end_time: new Date().getTime(),
       }),
     ];
 
@@ -397,10 +403,12 @@ describe("batch export test suite", () => {
       expect.arrayContaining([
         expect.objectContaining({
           id: traces[0].id,
+          latency: 1,
           test: [score.value],
         }),
         expect.objectContaining({
           id: traces[1].id,
+          latency: 2.123,
         }),
       ]),
     );

worker/src/__tests__/evalService.test.ts

@@ -763,8 +763,8 @@ describe("eval service tests", () => {
         id: traceId,
         project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         user_id: "a",
-        input: { input: "This is a great prompt" },
-        output: { output: "This is a great response" },
+        input: JSON.stringify({ input: "This is a great prompt" }),
+        output: JSON.stringify({ output: "This is a great response" }),
         timestamp: convertDateToClickhouseDateTime(new Date()),
         created_at: convertDateToClickhouseDateTime(new Date()),
         updated_at: convertDateToClickhouseDateTime(new Date()),
@@ -1233,8 +1233,8 @@ describe("eval service tests", () => {
         id: traceId,
         project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         user_id: "a",
-        input: { input: "This is a great prompt" },
-        output: { output: "This is a great response" },
+        input: JSON.stringify({ input: "This is a great prompt" }),
+        output: JSON.stringify({ output: "This is a great response" }),
         timestamp: convertDateToClickhouseDateTime(new Date()),
         created_at: convertDateToClickhouseDateTime(new Date()),
         updated_at: convertDateToClickhouseDateTime(new Date()),
@@ -1290,8 +1290,8 @@ describe("eval service tests", () => {
         id: traceId,
         project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         user_id: "a",
-        input: { input: "This is a great prompt" },
-        output: { output: "This is a great response" },
+        input: JSON.stringify({ input: "This is a great prompt" }),
+        output: JSON.stringify({ output: "This is a great response" }),
         timestamp: convertDateToClickhouseDateTime(new Date()),
         created_at: convertDateToClickhouseDateTime(new Date()),
         updated_at: convertDateToClickhouseDateTime(new Date()),
@@ -1316,8 +1316,8 @@ describe("eval service tests", () => {
         project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         name: "great-llm-name",
         type: "GENERATION",
-        input: { huhu: "This is a great prompt" },
-        output: { haha: "This is a great response" },
+        input: JSON.stringify({ huhu: "This is a great prompt" }),
+        output: JSON.stringify({ haha: "This is a great response" }),
         start_time: convertDateToClickhouseDateTime(new Date()),
         created_at: convertDateToClickhouseDateTime(new Date()),
         updated_at: convertDateToClickhouseDateTime(new Date()),
@@ -1418,8 +1418,8 @@ describe("eval service tests", () => {
         id: traceId,
         project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         user_id: "a",
-        input: { input: "This is a great prompt" },
-        output: { output: "This is a great response" },
+        input: JSON.stringify({ input: "This is a great prompt" }),
+        output: JSON.stringify({ output: "This is a great response" }),
         timestamp: convertDateToClickhouseDateTime(new Date()),
         created_at: convertDateToClickhouseDateTime(new Date()),
         updated_at: convertDateToClickhouseDateTime(new Date()),
@@ -1500,8 +1500,8 @@ describe("eval service tests", () => {
         id: traceId,
         project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         user_id: "a",
-        input: { input: "This is a great prompt" },
-        output: { output: "This is a great response" },
+        input: JSON.stringify({ input: "This is a great prompt" }),
+        output: JSON.stringify({ output: "This is a great response" }),
         timestamp: convertDateToClickhouseDateTime(new Date()),
         created_at: convertDateToClickhouseDateTime(new Date()),
         updated_at: convertDateToClickhouseDateTime(new Date()),
@@ -1527,8 +1527,8 @@ describe("eval service tests", () => {
         project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         name: "great-llm-name",
         type: "GENERATION",
-        input: { huhu: "This is a great prompt" },
-        output: { haha: "This is a great response" },
+        input: JSON.stringify({ huhu: "This is a great prompt" }),
+        output: JSON.stringify({ haha: "This is a great response" }),
         start_time: convertDateToClickhouseDateTime(
           new Date("2022-01-01T00:00:00.000Z"),
         ),
@@ -1556,8 +1556,8 @@ describe("eval service tests", () => {
         project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         name: "great-llm-name",
         type: "GENERATION",
-        input: { huhu: "This is a great prompt again" },
-        output: { haha: "This is a great response again" },
+        input: JSON.stringify({ huhu: "This is a great prompt again" }),
+        output: JSON.stringify({ haha: "This is a great response again" }),
         start_time: convertDateToClickhouseDateTime(
           new Date("2022-01-02T00:00:00.000Z"),
         ),

worker/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v3.3.0";
+export const VERSION = "v3.4.0";

worker/src/features/batchExport/handleBatchExportJob.ts

@@ -401,6 +401,7 @@ export const getDatabaseReadStream = async ({
                 input: fullTrace?.input,
                 output: fullTrace?.output,
                 metadata: fullTrace?.metadata,
+                latency: metric?.latency,
                 name: t.name ?? "",
                 usage: {
                   promptTokens: metric?.promptTokens,