chore: release v3.78.1

* run experiments with placeholders as well

* add missing file

* fix experiements

* run outside of loop

* add tests

* fix tests

* make tests clean again

* also test that variables resovle first

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.78.0",
+  "version": "3.78.1",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/src/index.ts

@@ -13,6 +13,7 @@ export * from "./utils/json";
 export * from "./utils/stringChecks";
 export * from "./utils/objects";
 export * from "./utils/typeChecks";
+export * from "./utils/prompts";
 export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
 export * from "./tableDefinitions/typeHelpers";
@@ -45,7 +46,7 @@ export * from "./features/prompts/parsePromptDependencyTags";
 export * from "./features/prompts/validation";
 export * from "./features/prompts/types";
 export * from "./features/prompts/constants";
-export * from "./server/llm/compileChatMessages";
+export { compileChatMessages, compileChatMessagesWithIds, isPlaceholder, type MessagePlaceholderValues, type PromptMessage as ServerPromptMessage } from "./server/llm/compileChatMessages";
 
 // export db types only
 export * from "@prisma/client";

packages/shared/src/utils/prompts.ts

@@ -0,0 +1,24 @@
+/**
+ * Client-safe utility functions for prompt handling
+ */
+
+export interface PromptMessage {
+  type?: string;
+  name?: string;
+  role?: string;
+  content?: string;
+}
+
+/**
+ * Extracts placeholder names from prompt messages.
+ * This is a client-safe version that doesn't depend on server-side types.
+ * @param messages Array of prompt messages
+ * @returns Array of placeholder names
+ */
+export function extractPlaceholderNames(messages: PromptMessage[]): string[] {
+  return messages
+    .filter((msg): msg is PromptMessage & { name: string } =>
+      msg.type === "placeholder" && typeof msg.name === "string"
+    )
+    .map(msg => msg.name);
+}

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.78.0",
+  "version": "3.78.1",
   "private": true,
   "license": "MIT",
   "engines": {

web/src/components/deleteButton.tsx

@@ -338,6 +338,7 @@ export function DeleteEvalConfigButton(props: DeleteButtonProps) {
           source: isTableAction ? "table-single-row" : "eval config detail",
         })
       }
+      customDeletePrompt="This action cannot be undone and removes all logs associated with this running evaluator. Scores produced by this evaluator will not be deleted."
       entityToDeleteName="running evaluator"
       executeDeleteMutation={executeDeleteMutation}
       isDeleteMutationLoading={evaluatorMutation.isLoading}

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v3.78.0";
+export const VERSION = "v3.78.1";

web/src/features/experiments/components/CreateExperimentsForm.tsx

@@ -638,7 +638,7 @@ export const CreateExperimentsForm = ({
                                 <span>
                                   Given current prompt, dataset item input must
                                   contain at least one of these first-level JSON
-                                  keys, mapped to a string value:
+                                  keys:
                                 </span>
                                 <ul className="my-2 ml-2 list-inside list-disc">
                                   {expectedColumns.map((col) => (
@@ -646,8 +646,9 @@ export const CreateExperimentsForm = ({
                                   ))}
                                 </ul>
                                 <span>
-                                  These will be used as the input to your
-                                  prompt.
+                                  Variables (like {"{{variable}}"}) should be mapped to string values.
+                                  Placeholders should be mapped to arrays of message objects.
+                                  These will be used as the input to your prompt.
                                 </span>
                               </div>
                             ) : (
@@ -721,7 +722,7 @@ export const CreateExperimentsForm = ({
                       <Loader2 className="h-3 w-3 animate-spin" />
                     </CardTitle>
                     <CardDescription className="text-foreground">
-                      Checking dataset items against prompt variables
+                      Checking dataset items against prompt variables and placeholders
                     </CardDescription>
                   </CardHeader>
                 </Card>
@@ -747,7 +748,7 @@ export const CreateExperimentsForm = ({
                       <CircleCheck className="h-4 w-4" />
                     </CardTitle>
                     <div className="text-sm">
-                      Matches between dataset items and prompt variables
+                      Matches between dataset items and prompt variables/placeholders
                       <ul className="my-2 ml-2 list-inside list-disc">
                         {Object.entries(
                           validationResult.data.variablesMap ?? {},
@@ -760,7 +761,7 @@ export const CreateExperimentsForm = ({
                           </li>
                         ))}
                       </ul>
-                      Items missing all prompt variables will be excluded from
+                      Items missing all required variables and placeholders will be excluded from
                       the experiment.
                     </div>
                   </CardHeader>

web/src/features/experiments/hooks/useExperimentPromptData.ts

@@ -1,7 +1,7 @@
 import { useMemo } from "react";
 import { api } from "@/src/utils/api";
 import { type UseFormReturn } from "react-hook-form";
-import { extractVariables, PromptType } from "@langfuse/shared";
+import { extractVariables, PromptType, extractPlaceholderNames, type PromptMessage } from "@langfuse/shared";
 
 type ExperimentPromptDataProps = {
   projectId: string;
@@ -22,11 +22,18 @@ export function useExperimentPromptData({
     const prompt = promptMeta.data?.find((p) => p.id === promptId);
     if (!prompt) return [];
 
-    return extractVariables(
+    const extractedVariables = extractVariables(
       prompt.type === PromptType.Text
         ? (prompt?.prompt?.toString() ?? "")
         : JSON.stringify(prompt?.prompt),
     );
+
+    const promptMessages = prompt?.type === PromptType.Chat && Array.isArray(prompt.prompt)
+      ? prompt.prompt
+      : [];
+    const placeholderNames = extractPlaceholderNames(promptMessages as PromptMessage[]);
+
+    return [...extractedVariables, ...placeholderNames];
   }, [promptId, promptMeta.data]);
 
   const promptsByName = useMemo(

web/src/features/experiments/server/router.ts

@@ -2,12 +2,11 @@ import { z } from "zod/v4";
 import { randomUUID } from "crypto";
 import {
   type ExperimentMetadata,
+  ExperimentCreateQueue,
   QueueJobs,
   QueueName,
   redis,
   ZodModelConfig,
-  ExperimentCreateQueue,
-  type PlaceholderMessage,
 } from "@langfuse/shared/src/server";
 import {
   createTRPCRouter,
@@ -20,6 +19,8 @@ import {
   datasetItemMatchesVariable,
   UnauthorizedError,
   PromptType,
+  extractPlaceholderNames,
+  type PromptMessage,
 } from "@langfuse/shared";
 import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
 
@@ -98,24 +99,17 @@ export const experimentsRouter = createTRPCRouter({
           : JSON.stringify(prompt.prompt),
       );
 
-      if (!Boolean(extractedVariables.length)) {
-        return {
-          isValid: false,
-          message: "Selected prompt has no variables.",
-        };
-      }
-
       const promptMessages = prompt?.type === PromptType.Chat && Array.isArray(prompt.prompt)
         ? prompt.prompt
         : [];
-      const hasPlaceholders = promptMessages.some((msg): msg is PlaceholderMessage => 
-        (msg as PlaceholderMessage).type === "placeholder"
-      );
+      const placeholderNames = extractPlaceholderNames(promptMessages as PromptMessage[]);
 
-      if (hasPlaceholders) {
+      const allVariables = [...extractedVariables, ...placeholderNames];
+
+      if (!Boolean(allVariables.length)) {
         return {
           isValid: false,
-          message: "Selected prompt has placeholders, those are not yet supported for experiments.",
+          message: "Selected prompt has no variables or placeholders.",
         };
       }
 
@@ -136,7 +130,7 @@ export const experimentsRouter = createTRPCRouter({
 
       const variablesMap = validateDatasetItems(
         datasetItems,
-        extractedVariables,
+        allVariables,
       );
 
       if (!Boolean(Object.keys(variablesMap).length)) {

web/src/features/prompts/components/prompts-table.tsx

@@ -484,7 +484,7 @@ export function PromptTable() {
         setFilterState={useDebounce(setFilterState)}
         columnsWithCustomSelect={["labels", "tags"]}
         searchConfig={{
-          metadataSearchFields: ["Name"],
+          metadataSearchFields: ["Name", "Tags"],
           updateQuery: useDebounce(setSearchQuery, 300),
           currentQuery: searchQuery ?? undefined,
           tableAllowsFullTextSearch: false,

web/src/features/prompts/server/routers/promptRouter.ts

@@ -92,7 +92,7 @@ export const promptRouter = createTRPCRouter({
         : Prisma.empty;
 
       const searchFilter = input.searchQuery
-        ? Prisma.sql` AND p.name ILIKE ${`%${input.searchQuery}%`}`
+        ? Prisma.sql` AND (p.name ILIKE ${`%${input.searchQuery}%`} OR EXISTS (SELECT 1 FROM UNNEST(p.tags) AS tag WHERE tag ILIKE ${`%${input.searchQuery}%`}))`
         : Prisma.empty;
 
       const [prompts, promptCount] = await Promise.all([
@@ -119,19 +119,19 @@ export const promptRouter = createTRPCRouter({
             searchFilter,
           ),
         ),
-                  // promptCount
-          ctx.prisma.$queryRaw<Array<{ totalCount: bigint }>>(
-            generatePromptQuery(
-              Prisma.sql` count(*) AS "totalCount"`,
-              input.projectId,
-              filterCondition,
-              Prisma.empty,
-              1, // limit
-              0, // page,
-              pathFilter,
-              searchFilter,
-            ),
+        // promptCount
+        ctx.prisma.$queryRaw<Array<{ totalCount: bigint }>>(
+          generatePromptQuery(
+            Prisma.sql` count(*) AS "totalCount"`,
+            input.projectId,
+            filterCondition,
+            Prisma.empty,
+            1, // limit
+            0, // page,
+            pathFilter,
+            searchFilter,
           ),
+        ),
       ]);
 
       return {

web/src/features/widgets/components/WidgetForm.tsx

@@ -48,15 +48,14 @@ import {
   Hash,
   BarChart3,
   Table,
-  X,
   Plus,
+  X,
 } from "lucide-react";
 import {
   buildWidgetName,
   buildWidgetDescription,
   formatMetricName,
 } from "@/src/features/widgets/utils";
-import { Badge } from "@/src/components/ui/badge";
 import {
   MAX_PIVOT_TABLE_DIMENSIONS,
   MAX_PIVOT_TABLE_METRICS,
@@ -284,40 +283,74 @@ export function WidgetForm({
     setPivotDimensions(newDimensions);
   };
 
-  // Helper functions for multiple metrics management
-  const addMetric = () => {
-    // Check if we've reached the maximum metrics limit
-    if (selectedMetrics.length >= MAX_PIVOT_TABLE_METRICS) {
-      showErrorToast(
-        "Error",
-        `Maximum ${MAX_PIVOT_TABLE_METRICS} metrics allowed for pivot tables`,
-      );
-      return;
+  // Helper function for updating pivot table metrics
+  const updatePivotMetric = (
+    index: number,
+    measure: string,
+    aggregation?: z.infer<typeof metricAggregations>,
+  ) => {
+    const newMetrics = [...selectedMetrics];
+
+    if (measure && measure !== "none") {
+      let finalAggregation: z.infer<typeof metricAggregations>;
+
+      if (measure === "count") {
+        finalAggregation = "count";
+      } else {
+        // Get available aggregations for this measure at this index
+        const availableAggregations = getAvailableAggregations(index, measure);
+
+        if (aggregation && availableAggregations.includes(aggregation)) {
+          // Use provided aggregation if it's available
+          finalAggregation = aggregation as z.infer<typeof metricAggregations>;
+        } else {
+          // Use the first available aggregation as default
+          finalAggregation =
+            availableAggregations.length > 0
+              ? availableAggregations[0]
+              : ("sum" as z.infer<typeof metricAggregations>);
+        }
+      }
+
+      const newMetric: SelectedMetric = {
+        id: `${finalAggregation}_${measure}`,
+        measure: measure,
+        aggregation: finalAggregation as z.infer<typeof metricAggregations>,
+        label: `${startCase(finalAggregation)} ${startCase(measure)}`,
+      };
+
+      // Set the metric at the specified index
+      newMetrics[index] = newMetric;
+    } else {
+      // Clear this metric and all subsequent ones
+      newMetrics.splice(index);
     }
 
-    const newMetric: SelectedMetric = {
-      id: `${selectedAggregation}_${selectedMeasure}`,
-      measure: selectedMeasure,
-      aggregation: selectedAggregation,
-      label: `${startCase(selectedAggregation)} ${startCase(selectedMeasure)}`,
-    };
+    setSelectedMetrics(newMetrics);
+  };
 
-    // Check if this metric combination already exists
-    const existingMetric = selectedMetrics.find((m) => m.id === newMetric.id);
-    if (existingMetric) {
-      showErrorToast("Error", "This metric combination is already selected");
-      return;
+  // Add a new empty metric slot
+  const addNewMetricSlot = () => {
+    if (selectedMetrics.length < MAX_PIVOT_TABLE_METRICS) {
+      const newMetrics = [...selectedMetrics];
+      newMetrics.push({
+        id: `temp_${selectedMetrics.length}`,
+        measure: "",
+        aggregation: "sum" as z.infer<typeof metricAggregations>,
+        label: "",
+      });
+      setSelectedMetrics(newMetrics);
     }
-
-    setSelectedMetrics([...selectedMetrics, newMetric]);
   };
 
-  const removeMetric = (metricId: string) => {
-    setSelectedMetrics(selectedMetrics.filter((m) => m.id !== metricId));
-  };
-
-  const clearAllMetrics = () => {
-    setSelectedMetrics([]);
+  // Remove a metric slot and roll up subsequent ones
+  const removeMetricSlot = (index: number) => {
+    if (index > 0) {
+      // Can't remove the first metric (it's required)
+      const newMetrics = [...selectedMetrics];
+      newMetrics.splice(index, 1); // Remove only the metric at this index
+      setSelectedMetrics(newMetrics);
+    }
   };
 
   const traceFilterOptions = api.traces.filterOptions.useQuery(
@@ -542,6 +575,40 @@ export function WidgetForm({
   // Get available metrics for the selected view
   const availableMetrics = useMemo(() => {
     const viewDeclaration = viewDeclarations[selectedView];
+
+    // For pivot tables, only show measures that still have available aggregations
+    if (selectedChartType === "PIVOT_TABLE") {
+      return Object.entries(viewDeclaration.measures)
+        .filter(([measureKey]) => {
+          // For count, there's only one aggregation option
+          if (measureKey === "count") {
+            return !selectedMetrics.some((m) => m.measure === "count");
+          }
+
+          // For other measures, check if there are any aggregations left
+          const selectedAggregationsForMeasure = selectedMetrics
+            .filter((m) => m.measure === measureKey)
+            .map((m) => m.aggregation);
+
+          const availableAggregationsForMeasure =
+            metricAggregations.options.filter(
+              (agg) =>
+                agg !== "histogram" &&
+                !selectedAggregationsForMeasure.includes(agg),
+            );
+
+          return availableAggregationsForMeasure.length > 0;
+        })
+        .map(([key]) => ({
+          value: key,
+          label: startCase(key),
+        }))
+        .sort((a, b) =>
+          a.label.localeCompare(b.label, "en", { sensitivity: "base" }),
+        );
+    }
+
+    // For regular charts, show all metrics
     return Object.entries(viewDeclaration.measures)
       .map(([key]) => ({
         value: key,
@@ -550,7 +617,64 @@ export function WidgetForm({
       .sort((a, b) =>
         a.label.localeCompare(b.label, "en", { sensitivity: "base" }),
       );
-  }, [selectedView]);
+  }, [selectedView, selectedChartType, selectedMetrics]);
+
+  // Get available aggregations for a specific metric index in pivot tables
+  const getAvailableAggregations = (
+    metricIndex: number,
+    measureKey: string,
+  ): z.infer<typeof metricAggregations>[] => {
+    if (selectedChartType === "PIVOT_TABLE" && measureKey) {
+      return metricAggregations.options.filter(
+        (agg) =>
+          !selectedMetrics.some(
+            (m, idx) =>
+              idx !== metricIndex &&
+              m.measure === measureKey &&
+              m.aggregation === agg,
+          ),
+      ) as z.infer<typeof metricAggregations>[];
+    }
+    return metricAggregations.options as z.infer<typeof metricAggregations>[];
+  };
+
+  // Get available metrics for a specific metric index in pivot tables
+  const getAvailableMetrics = (metricIndex: number) => {
+    if (selectedChartType === "PIVOT_TABLE") {
+      const viewDeclaration = viewDeclarations[selectedView];
+      return Object.entries(viewDeclaration.measures)
+        .filter(([measureKey]) => {
+          // For count, there's only one aggregation option
+          if (measureKey === "count") {
+            return !selectedMetrics.some(
+              (m, idx) => idx !== metricIndex && m.measure === "count",
+            );
+          }
+
+          // For other measures, check if there are any aggregations left
+          const selectedAggregationsForMeasure = selectedMetrics
+            .filter((m, idx) => idx !== metricIndex && m.measure === measureKey)
+            .map((m) => m.aggregation);
+
+          const availableAggregationsForMeasure =
+            metricAggregations.options.filter(
+              (agg) =>
+                agg !== "histogram" &&
+                !selectedAggregationsForMeasure.includes(agg),
+            );
+
+          return availableAggregationsForMeasure.length > 0;
+        })
+        .map(([key]) => ({
+          value: key,
+          label: startCase(key),
+        }))
+        .sort((a, b) =>
+          a.label.localeCompare(b.label, "en", { sensitivity: "base" }),
+        );
+    }
+    return availableMetrics;
+  };
 
   // Get available dimensions for the selected view
   const availableDimensions = useMemo(() => {
@@ -584,10 +708,12 @@ export function WidgetForm({
     // Determine metrics based on chart type
     const queryMetrics =
       selectedChartType === "PIVOT_TABLE"
-        ? selectedMetrics.map((metric) => ({
-            measure: metric.measure,
-            aggregation: metric.aggregation,
-          }))
+        ? selectedMetrics
+            .filter((metric) => metric.measure && metric.measure !== "")
+            .map((metric) => ({
+              measure: metric.measure,
+              aggregation: metric.aggregation,
+            }))
         : [
             {
               measure: selectedMeasure,
@@ -701,7 +827,10 @@ export function WidgetForm({
     }
 
     // Validate pivot table requirements
-    if (selectedChartType === "PIVOT_TABLE" && selectedMetrics.length === 0) {
+    const validMetrics = selectedMetrics.filter(
+      (m) => m.measure && m.measure !== "",
+    );
+    if (selectedChartType === "PIVOT_TABLE" && validMetrics.length === 0) {
       showErrorToast(
         "Error",
         "At least one metric is required for pivot tables",
@@ -721,7 +850,7 @@ export function WidgetForm({
             : [],
       metrics:
         selectedChartType === "PIVOT_TABLE"
-          ? selectedMetrics.map((metric) => ({
+          ? validMetrics.map((metric) => ({
               measure: metric.measure,
               agg: metric.aggregation,
             }))
@@ -766,9 +895,13 @@ export function WidgetForm({
 
     // For pivot tables, extract actual metric names for the new formatting
     const isPivotTable = selectedChartType === "PIVOT_TABLE";
+
+    const validMetricsForNaming = selectedMetrics.filter(
+      (m) => m.measure && m.measure !== "",
+    );
     const metricNames =
-      isPivotTable && selectedMetrics.length > 0
-        ? selectedMetrics.map((m) => m.id) // Use the ID which is "${aggregation}_${measure}"
+      isPivotTable && validMetricsForNaming.length > 0
+        ? validMetricsForNaming.map((m) => m.id) // Use the ID which is "${aggregation}_${measure}"
         : undefined;
 
     const suggested = buildWidgetName({
@@ -777,7 +910,7 @@ export function WidgetForm({
       dimension: dimensionForNaming,
       view: selectedView,
       metrics: metricNames,
-      isMultiMetric: isPivotTable && selectedMetrics.length > 0,
+      isMultiMetric: isPivotTable && validMetricsForNaming.length > 0,
     });
 
     setWidgetName(suggested);
@@ -804,9 +937,12 @@ export function WidgetForm({
 
     // For pivot tables, extract actual metric names for the new formatting
     const isPivotTable = selectedChartType === "PIVOT_TABLE";
+    const validMetricsForDescription = selectedMetrics.filter(
+      (m) => m.measure && m.measure !== "",
+    );
     const metricNames =
-      isPivotTable && selectedMetrics.length > 0
-        ? selectedMetrics.map((m) => m.id) // Use the ID which is "${aggregation}_${measure}"
+      isPivotTable && validMetricsForDescription.length > 0
+        ? validMetricsForDescription.map((m) => m.id) // Use the ID which is "${aggregation}_${measure}"
         : undefined;
 
     const suggested = buildWidgetDescription({
@@ -816,7 +952,7 @@ export function WidgetForm({
       view: selectedView,
       filters: userFilterState,
       metrics: metricNames,
-      isMultiMetric: isPivotTable && selectedMetrics.length > 0,
+      isMultiMetric: isPivotTable && validMetricsForDescription.length > 0,
     });
 
     setWidgetDescription(suggested);
@@ -887,118 +1023,146 @@ export function WidgetForm({
                 {/* For pivot tables: multiple metrics selection */}
                 {selectedChartType === "PIVOT_TABLE" ? (
                   <div className="space-y-3">
-                    {/* Selected metrics display */}
-                    {selectedMetrics.length > 0 && (
-                      <div className="space-y-2">
-                        <div className="flex items-center justify-between">
-                          <span className="text-sm font-medium">
-                            Selected Metrics ({selectedMetrics.length})
-                          </span>
-                          <Button
-                            type="button"
-                            variant="ghost"
-                            size="sm"
-                            onClick={clearAllMetrics}
-                            className="h-auto p-1 text-xs"
-                          >
-                            Clear All
-                          </Button>
-                        </div>
-                        <div className="flex flex-wrap gap-1">
-                          {selectedMetrics.map((metric) => (
-                            <Badge
-                              key={metric.id}
-                              variant="secondary"
-                              className="flex items-center gap-1"
-                            >
-                              <span className="text-xs">{metric.label}</span>
-                              <Button
-                                type="button"
-                                variant="ghost"
-                                size="sm"
-                                onClick={() => removeMetric(metric.id)}
-                                className="h-auto p-0 hover:bg-transparent"
-                              >
-                                <X className="h-3 w-3" />
-                              </Button>
-                            </Badge>
-                          ))}
-                        </div>
-                      </div>
+                    {/* Metric selection dropdowns */}
+                    {Array.from(
+                      { length: Math.max(1, selectedMetrics.length) },
+                      (_, index) => {
+                        const isEnabled =
+                          index === 0 ||
+                          (selectedMetrics[index - 1] &&
+                            selectedMetrics[index - 1].measure);
+                        const currentMetric = selectedMetrics[index];
+                        const currentMeasure = currentMetric?.measure || "";
+                        const currentAggregation =
+                          currentMetric?.aggregation || "sum";
+
+                        const metricsForIndex = getAvailableMetrics(index);
+                        const aggregationsForIndex = getAvailableAggregations(
+                          index,
+                          currentMeasure,
+                        );
+
+                        const canEdit = metricsForIndex.length > 0;
+
+                        return (
+                          <div key={index} className="space-y-2">
+                            <div className="flex items-center justify-between">
+                              <Label htmlFor={`pivot-metric-${index}`}>
+                                Metric {index + 1}{" "}
+                                {index === 0 ? "(Required)" : "(Optional)"}
+                              </Label>
+                              {index > 0 && (
+                                <Button
+                                  type="button"
+                                  variant="ghost"
+                                  size="sm"
+                                  onClick={() => removeMetricSlot(index)}
+                                  className="h-6 w-6 p-0 text-muted-foreground hover:text-destructive"
+                                >
+                                  <X className="h-4 w-4" />
+                                </Button>
+                              )}
+                            </div>
+                            <div className="flex items-center gap-2">
+                              <div className="flex-1">
+                                <Select
+                                  value={currentMeasure}
+                                  onValueChange={(value) =>
+                                    updatePivotMetric(
+                                      index,
+                                      value,
+                                      // Don't pass current aggregation when measure changes
+                                      // Let the function determine the best default
+                                      undefined,
+                                    )
+                                  }
+                                  disabled={!isEnabled || !canEdit}
+                                >
+                                  <SelectTrigger id={`pivot-metric-${index}`}>
+                                    <SelectValue
+                                      placeholder={
+                                        !isEnabled
+                                          ? "Select previous metric first"
+                                          : !canEdit
+                                            ? "No more measures available"
+                                            : "Select measure"
+                                      }
+                                    />
+                                  </SelectTrigger>
+                                  <SelectContent>
+                                    {metricsForIndex.map((metric) => {
+                                      const meta =
+                                        viewDeclarations[selectedView]
+                                          ?.measures?.[metric.value];
+                                      return (
+                                        <WidgetPropertySelectItem
+                                          key={metric.value}
+                                          value={metric.value}
+                                          label={metric.label}
+                                          description={meta?.description}
+                                          unit={meta?.unit}
+                                          type={meta?.type}
+                                        />
+                                      );
+                                    })}
+                                  </SelectContent>
+                                </Select>
+                              </div>
+
+                              {currentMeasure &&
+                                currentMeasure !== "count" && (
+                                  <div className="flex-1">
+                                    <Select
+                                      value={currentAggregation}
+                                      onValueChange={(value) =>
+                                        updatePivotMetric(
+                                          index,
+                                          currentMeasure,
+                                          value as z.infer<
+                                            typeof metricAggregations
+                                          >,
+                                        )
+                                      }
+                                    >
+                                      <SelectTrigger>
+                                        <SelectValue placeholder="Select aggregation" />
+                                      </SelectTrigger>
+                                      <SelectContent>
+                                        {aggregationsForIndex.map(
+                                          (aggregation) => (
+                                            <SelectItem
+                                              key={aggregation}
+                                              value={aggregation}
+                                            >
+                                              {startCase(aggregation)}
+                                            </SelectItem>
+                                          ),
+                                        )}
+                                      </SelectContent>
+                                    </Select>
+                                  </div>
+                                )}
+                            </div>
+                          </div>
+                        );
+                      },
                     )}
 
-                    {/* Add new metric controls */}
-                    <div className="space-y-2 rounded-md border p-3">
-                      <span className="text-sm font-medium">Add Metric</span>
-                      <div className="grid grid-cols-2 gap-2">
-                        <Select
-                          value={selectedMeasure}
-                          onValueChange={(value) => setSelectedMeasure(value)}
+                    {/* Add new metric button */}
+                    {selectedMetrics.length < MAX_PIVOT_TABLE_METRICS &&
+                      getAvailableMetrics(selectedMetrics.length).length >
+                        0 && (
+                        <Button
+                          type="button"
+                          variant="outline"
+                          size="sm"
+                          onClick={addNewMetricSlot}
+                          className="w-full"
                         >
-                          <SelectTrigger>
-                            <SelectValue placeholder="Select measure" />
-                          </SelectTrigger>
-                          <SelectContent>
-                            {availableMetrics.map((metric) => {
-                              const meta =
-                                viewDeclarations[selectedView]?.measures?.[
-                                  metric.value
-                                ];
-                              return (
-                                <WidgetPropertySelectItem
-                                  key={metric.value}
-                                  value={metric.value}
-                                  label={metric.label}
-                                  description={meta?.description}
-                                  unit={meta?.unit}
-                                  type={meta?.type}
-                                />
-                              );
-                            })}
-                          </SelectContent>
-                        </Select>
-
-                        {selectedMeasure !== "count" && (
-                          <Select
-                            value={selectedAggregation}
-                            onValueChange={(value) =>
-                              setSelectedAggregation(
-                                value as z.infer<typeof metricAggregations>,
-                              )
-                            }
-                          >
-                            <SelectTrigger>
-                              <SelectValue placeholder="Aggregation" />
-                            </SelectTrigger>
-                            <SelectContent>
-                              {metricAggregations.options.map((aggregation) => (
-                                <SelectItem
-                                  key={aggregation}
-                                  value={aggregation}
-                                >
-                                  {startCase(aggregation)}
-                                </SelectItem>
-                              ))}
-                            </SelectContent>
-                          </Select>
-                        )}
-                      </div>
-                      <Button
-                        type="button"
-                        variant="outline"
-                        size="sm"
-                        onClick={addMetric}
-                        className="w-full"
-                        disabled={
-                          selectedMetrics.length >= MAX_PIVOT_TABLE_METRICS
-                        }
-                      >
-                        <Plus className="mr-1 h-3 w-3" />
-                        Add Metric
-                        {selectedMetrics.length >= MAX_PIVOT_TABLE_METRICS &&
-                          ` (Max ${MAX_PIVOT_TABLE_METRICS})`}
-                      </Button>
-                    </div>
+                          <Plus className="mr-1 h-3 w-3" />
+                          Add Metric {selectedMetrics.length + 1}
+                        </Button>
+                      )}
                   </div>
                 ) : (
                   /* For regular charts: single metric selection */

worker/package.json

@@ -1,6 +1,6 @@
 {
   "name": "worker",
-  "version": "3.78.0",
+  "version": "3.78.1",
   "description": "",
   "license": "MIT",
   "private": true,

worker/src/__tests__/experimentsService.test.ts

@@ -313,6 +313,187 @@ describe("create experiment jobs", () => {
   }, 10_000);
 });
 
+describe("create experiment jobs with placeholders", () => {
+  const setupPlaceholderTest = async (promptConfig: any, datasetItemInput: any) => {
+    await pruneDatabase();
+    const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+    const datasetId = randomUUID();
+    const runId = randomUUID();
+    const promptId = "03f834cc-c089-4bcb-9add-b14cadcdf47c";
+
+    // Create prompt
+    await prisma.prompt.create({
+      data: {
+        id: promptId,
+        projectId,
+        name: promptConfig.name,
+        prompt: promptConfig.prompt,
+        type: "chat",
+        version: 1,
+        createdBy: "test-user",
+      },
+    });
+
+    // Create dataset
+    await prisma.dataset.create({
+      data: {
+        id: datasetId,
+        projectId,
+        name: "Test Dataset",
+      },
+    });
+
+    // Create dataset run with metadata
+    await kyselyPrisma.$kysely
+      .insertInto("dataset_runs")
+      .values({
+        id: runId,
+        name: "Test Run",
+        project_id: projectId,
+        dataset_id: datasetId,
+        metadata: {
+          prompt_id: promptId,
+          provider: "openai",
+          model: "gpt-3.5-turbo",
+          model_params: { temperature: 0 },
+        },
+      })
+      .execute();
+
+    // Create dataset item
+    await prisma.datasetItem.create({
+      data: {
+        id: randomUUID(),
+        projectId,
+        datasetId,
+        input: datasetItemInput,
+      },
+    });
+
+    // Create API key
+    await prisma.llmApiKeys.create({
+      data: {
+        id: randomUUID(),
+        projectId,
+        provider: "openai",
+        adapter: LLMAdapter.OpenAI,
+        displaySecretKey: "test-key",
+        secretKey: encrypt("test-key"),
+      },
+    });
+
+    return { projectId, datasetId, runId };
+  };
+
+  test("creates experiment job with multiple placeholders containing variables", async () => {
+    const { projectId, datasetId, runId } = await setupPlaceholderTest(
+      {
+        name: "Test Multiple Placeholders",
+        prompt: [
+          { role: "system", content: "You are a helpful assistant." },
+          { type: "placeholder", name: "conversation_history" },
+          { type: "placeholder", name: "user_context" },
+          { role: "user", content: "Please help me." }
+        ]
+      },
+      {
+        conversation_history: [
+          { role: "user", content: "Hello {{name}}!" },
+          { role: "assistant", content: "Hi there!" }
+        ],
+        user_context: [
+          { role: "system", content: "User is a {{role}}" }
+        ],
+        name: "John",
+        role: "developer"
+      }
+    );
+
+    const payload = {
+      projectId,
+      datasetId,
+      runId,
+    };
+
+    await createExperimentJob({ event: payload });
+
+    const runItems = await kyselyPrisma.$kysely
+      .selectFrom("dataset_run_items")
+      .selectAll()
+      .where("project_id", "=", projectId)
+      .execute();
+
+    expect(runItems.length).toBe(1);
+    expect(runItems[0].project_id).toBe(projectId);
+    expect(runItems[0].dataset_run_id).toBe(runId);
+    expect(runItems[0].trace_id).toBeDefined();
+  }, 10_000);
+
+  test("handles empty placeholder arrays", async () => {
+    const { projectId, datasetId, runId } = await setupPlaceholderTest(
+      {
+        name: "Test Empty Placeholder",
+        prompt: [
+          { role: "system", content: "You are a helpful assistant." },
+          { type: "placeholder", name: "empty_history" },
+          { role: "user", content: "Start conversation." }
+        ]
+      },
+      { empty_history: [] }
+    );
+
+    const payload = {
+      projectId,
+      datasetId,
+      runId,
+    };
+
+    await createExperimentJob({ event: payload });
+
+    const runItems = await kyselyPrisma.$kysely
+      .selectFrom("dataset_run_items")
+      .selectAll()
+      .where("project_id", "=", projectId)
+      .execute();
+
+    expect(runItems.length).toBe(1);
+    expect(runItems[0].project_id).toBe(projectId);
+    expect(runItems[0].dataset_run_id).toBe(runId);
+    expect(runItems[0].trace_id).toBeDefined();
+  }, 10_000);
+
+  test("fails when placeholder has invalid message format", async () => {
+    const { projectId, datasetId, runId } = await setupPlaceholderTest(
+      {
+        name: "Test Invalid Placeholder",
+        prompt: [
+          { role: "system", content: "You are a helpful assistant." },
+          { type: "placeholder", name: "invalid_messages" },
+          { role: "user", content: "Help me." }
+        ]
+      },
+      { invalid_messages: "this should be an array or object" }
+    );
+
+    const payload = {
+      projectId,
+      datasetId,
+      runId,
+    };
+
+    await createExperimentJob({ event: payload });
+
+    // Should not create run items for invalid placeholder format
+    const runItems = await kyselyPrisma.$kysely
+      .selectFrom("dataset_run_items")
+      .selectAll()
+      .where("project_id", "=", projectId)
+      .execute();
+
+    expect(runItems.length).toBe(0);
+  }, 10_000);
+});
+
 describe("create experiment job calls with langfuse server side tracing", async () => {
   await pruneDatabase();
   const mockEvent = {

worker/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v3.78.0";
+export const VERSION = "v3.78.1";

worker/src/features/experiments/experimentService.ts

@@ -18,11 +18,12 @@ import {
 import { kyselyPrisma, prisma } from "@langfuse/shared/src/db";
 import { type ExperimentCreateEventSchema } from "@langfuse/shared/src/server";
 import {
+  datasetItemMatchesVariable,
+  extractVariables,
   InvalidRequestError,
   LangfuseNotFoundError,
   type Prisma,
-  extractVariables,
-  datasetItemMatchesVariable,
+  PromptType,
   stringifyValue,
 } from "@langfuse/shared";
 import { backOff } from "exponential-backoff";
@@ -45,11 +46,14 @@ const replaceVariablesInPrompt = (
   prompt: PromptContent,
   itemInput: Record<string, any>,
   variables: string[],
+  placeholderNames: string[] = [],
 ): ChatMessage[] => {
   const processContent = (content: string) => {
-    // Extract only relevant variables from itemInput
+    // Extract only Handlebars variables from itemInput (exclude message placeholders)
     const filteredContext = Object.fromEntries(
-      Object.entries(itemInput).filter(([key]) => variables.includes(key)),
+      Object.entries(itemInput).filter(([key]) =>
+        variables.includes(key) && !placeholderNames.includes(key)
+      ),
     );
 
     // Apply Handlebars ONLY if the content contains `{{variable}}` pattern
@@ -70,14 +74,12 @@ const replaceVariablesInPrompt = (
     ];
   }
 
-  const placeholderNames = extractPlaceholderNames(prompt as PromptMessage[]);
   const placeholderValues: MessagePlaceholderValues = {};
   // itemInput to placeholderValues
   for (const placeholderName of placeholderNames) {
     if (!(placeholderName in itemInput)) {
-      // TODO: handle missing placeholder values
-      // throw new Error(`Missing placeholder value for '${placeholderName}'`);
-      continue;
+      // TODO: should we throw?
+      throw new Error(`Missing placeholder value for '${placeholderName}'`);
     }
     const value = itemInput[placeholderName];
 
@@ -117,8 +119,6 @@ const replaceVariablesInPrompt = (
     {}
   );
 
-  // TODO: validate correctness
-  // handlebars variable substitution to all messages
   return compiledMessages.map((message) => ({
     ...message,
     content: processContent(message.content),
@@ -251,13 +251,13 @@ export const createExperimentJob = async ({
 
   // extract variables from prompt
   const extractedVariables = extractVariables(
-    prompt?.type === "text"
+    prompt?.type === PromptType.Text
       ? (prompt.prompt?.toString() ?? "")
       : JSON.stringify(prompt.prompt),
   );
 
-  // also extract placeholder names if prompt is an array
-  const placeholderNames = prompt?.type !== "text" && Array.isArray(validatedPrompt.data)
+  // also extract placeholder names if prompt is a chat prompt
+  const placeholderNames = prompt?.type === PromptType.Chat && Array.isArray(validatedPrompt.data)
     ? extractPlaceholderNames(validatedPrompt.data as PromptMessage[])
     : [];
   const allVariables = [...extractedVariables, ...placeholderNames];
@@ -275,7 +275,7 @@ export const createExperimentJob = async ({
 
   if (!validatedDatasetItems.length) {
     throw new InvalidRequestError(
-      `No Dataset ${datasetId} item input matches expected prompt variable format`,
+      `No Dataset ${datasetId} item input matches expected prompt variables or placeholders format`,
     );
   }
 
@@ -305,7 +305,8 @@ export const createExperimentJob = async ({
       messages = replaceVariablesInPrompt(
         validatedPrompt.data,
         datasetItem.input, // validated format
-        extractedVariables,
+        allVariables,
+        placeholderNames,
       );
     } catch (error) {
       // skip this dataset item if there is an error replacing variables