chore: release v2.7.0

Co-authored-by: Max Deichmann <m.deichmann@tum.de>

fern/apis/server/definition/commons.yml

@@ -29,7 +29,7 @@ types:
         docs: Latency of trace in seconds
       totalCost:
         type: double
-        docs: Cost of trace in USD.
+        docs: Cost of trace in USD
       observations:
         type: list<string>
         docs: List of observation ids
@@ -39,6 +39,12 @@ types:
   TraceWithFullDetails: # GET traces/[traceID]
     extends: Trace
     properties:
+      htmlPath:
+        type: string
+        docs: Path of trace in Langfuse UI
+      totalCost:
+        type: double
+        docs: Cost of trace in USD
       observations: list<ObservationsView>
       scores: list<Score>
   Session:

fern/apis/server/definition/prompts.yml

@@ -29,8 +29,11 @@ types:
       name: string
       isActive: boolean
       prompt: string
+      config: optional<unknown>
+
   Prompt:
     properties:
       name: string
       version: integer
       prompt: string
+      config: unknown

generated/openapi-server/openapi.yml

@@ -1110,7 +1110,7 @@ components:
         totalCost:
           type: number
           format: double
-          description: Cost of trace in USD.
+          description: Cost of trace in USD
         observations:
           type: array
           items:
@@ -1133,6 +1133,13 @@ components:
       title: TraceWithFullDetails
       type: object
       properties:
+        htmlPath:
+          type: string
+          description: Path of trace in Langfuse UI
+        totalCost:
+          type: number
+          format: double
+          description: Cost of trace in USD
         observations:
           type: array
           items:
@@ -1142,6 +1149,8 @@ components:
           items:
             $ref: '#/components/schemas/Score'
       required:
+        - htmlPath
+        - totalCost
         - observations
         - scores
       allOf:
@@ -2186,6 +2195,8 @@ components:
           type: boolean
         prompt:
           type: string
+        config:
+          nullable: true
       required:
         - name
         - isActive
@@ -2200,10 +2211,12 @@ components:
           type: integer
         prompt:
           type: string
+        config: {}
       required:
         - name
         - version
         - prompt
+        - config
     CreateScoreRequest:
       title: CreateScoreRequest
       type: object

generated/postman/collection.json

@@ -587,7 +587,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": \"example\",\n    \"config\": \"UNKNOWN\"\n}",
               "options": {
                 "raw": {
                   "language": "json"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse-core",
-  "version": "2.6.0",
+  "version": "2.7.0",
   "private": true,
   "scripts": {
     "prebuild": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
@@ -82,6 +82,7 @@
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
     "js-tiktoken": "^1.0.10",
+    "kysely": "^0.27.2",
     "lodash": "^4.17.21",
     "lucide-react": "^0.330.0",
     "next": "^14.1.0",
@@ -90,6 +91,7 @@
     "nodemailer": "^6.9.9",
     "posthog-js": "^1.105.9",
     "posthog-node": "^3.6.3",
+    "prisma-kysely": "^1.8.0",
     "react": "18.2.0",
     "react-day-picker": "^8.10.0",
     "react-dom": "18.2.0",

prisma/generated/types.ts

@@ -0,0 +1,339 @@
+import type { ColumnType } from "kysely";
+export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
+  ? ColumnType<S, I | undefined, U>
+  : ColumnType<T, T | undefined, T>;
+export type Timestamp = ColumnType<Date, Date | string, Date | string>;
+
+export const MembershipRole = {
+    OWNER: "OWNER",
+    ADMIN: "ADMIN",
+    MEMBER: "MEMBER",
+    VIEWER: "VIEWER"
+} as const;
+export type MembershipRole = (typeof MembershipRole)[keyof typeof MembershipRole];
+export const ObservationType = {
+    SPAN: "SPAN",
+    EVENT: "EVENT",
+    GENERATION: "GENERATION"
+} as const;
+export type ObservationType = (typeof ObservationType)[keyof typeof ObservationType];
+export const ObservationLevel = {
+    DEBUG: "DEBUG",
+    DEFAULT: "DEFAULT",
+    WARNING: "WARNING",
+    ERROR: "ERROR"
+} as const;
+export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
+export const PricingUnit = {
+    PER_1000_TOKENS: "PER_1000_TOKENS",
+    PER_1000_CHARS: "PER_1000_CHARS"
+} as const;
+export type PricingUnit = (typeof PricingUnit)[keyof typeof PricingUnit];
+export const TokenType = {
+    PROMPT: "PROMPT",
+    COMPLETION: "COMPLETION",
+    TOTAL: "TOTAL"
+} as const;
+export type TokenType = (typeof TokenType)[keyof typeof TokenType];
+export const DatasetStatus = {
+    ACTIVE: "ACTIVE",
+    ARCHIVED: "ARCHIVED"
+} as const;
+export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
+export type Account = {
+    id: string;
+    user_id: string;
+    type: string;
+    provider: string;
+    providerAccountId: string;
+    refresh_token: string | null;
+    access_token: string | null;
+    expires_at: number | null;
+    expires_in: number | null;
+    ext_expires_in: number | null;
+    token_type: string | null;
+    scope: string | null;
+    id_token: string | null;
+    session_state: string | null;
+};
+export type ApiKey = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    note: string | null;
+    public_key: string;
+    hashed_secret_key: string;
+    fast_hashed_secret_key: string | null;
+    display_secret_key: string;
+    last_used_at: Timestamp | null;
+    expires_at: Timestamp | null;
+    project_id: string;
+};
+export type AuditLog = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    user_id: string;
+    project_id: string;
+    user_project_role: MembershipRole;
+    resource_type: string;
+    resource_id: string;
+    action: string;
+    before: string | null;
+    after: string | null;
+};
+export type CronJobs = {
+    name: string;
+    last_run: Timestamp | null;
+    job_started_at: Timestamp | null;
+    state: string | null;
+};
+export type Dataset = {
+    id: string;
+    name: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
+export type DatasetItem = {
+    id: string;
+    status: Generated<DatasetStatus>;
+    input: unknown;
+    expected_output: unknown | null;
+    source_observation_id: string | null;
+    dataset_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
+export type DatasetRunItems = {
+    id: string;
+    dataset_run_id: string;
+    dataset_item_id: string;
+    observation_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
+export type DatasetRuns = {
+    id: string;
+    name: string;
+    dataset_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
+export type Events = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    data: unknown;
+    headers: Generated<unknown>;
+    url: string | null;
+    method: string | null;
+};
+export type Example = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Timestamp;
+};
+export type Membership = {
+    project_id: string;
+    user_id: string;
+    role: MembershipRole;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
+export type MembershipInvitation = {
+    id: string;
+    email: string;
+    role: MembershipRole;
+    project_id: string;
+    sender_id: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
+export type Model = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string | null;
+    model_name: string;
+    match_pattern: string;
+    start_date: Timestamp | null;
+    input_price: string | null;
+    output_price: string | null;
+    total_price: string | null;
+    unit: string;
+    tokenizer_id: string | null;
+    tokenizer_config: unknown | null;
+};
+export type Observation = {
+    id: string;
+    trace_id: string | null;
+    project_id: string;
+    type: ObservationType;
+    start_time: Generated<Timestamp>;
+    end_time: Timestamp | null;
+    name: string | null;
+    metadata: unknown | null;
+    parent_observation_id: string | null;
+    level: Generated<ObservationLevel>;
+    status_message: string | null;
+    version: string | null;
+    created_at: Generated<Timestamp>;
+    model: string | null;
+    internal_model: string | null;
+    modelParameters: unknown | null;
+    input: unknown | null;
+    output: unknown | null;
+    prompt_tokens: Generated<number>;
+    completion_tokens: Generated<number>;
+    total_tokens: Generated<number>;
+    unit: string | null;
+    input_cost: string | null;
+    output_cost: string | null;
+    total_cost: string | null;
+    completion_start_time: Timestamp | null;
+    prompt_id: string | null;
+};
+export type ObservationView = {
+    id: string;
+    trace_id: string | null;
+    project_id: string;
+    type: ObservationType;
+    start_time: Generated<Timestamp>;
+    end_time: Timestamp | null;
+    name: string | null;
+    metadata: unknown | null;
+    parent_observation_id: string | null;
+    level: Generated<ObservationLevel>;
+    status_message: string | null;
+    version: string | null;
+    created_at: Generated<Timestamp>;
+    model: string | null;
+    modelParameters: unknown | null;
+    input: unknown | null;
+    output: unknown | null;
+    prompt_tokens: Generated<number>;
+    completion_tokens: Generated<number>;
+    total_tokens: Generated<number>;
+    unit: string | null;
+    completion_start_time: Timestamp | null;
+    prompt_id: string | null;
+    model_id: string | null;
+    input_price: string | null;
+    output_price: string | null;
+    total_price: string | null;
+    calculated_input_cost: string | null;
+    calculated_output_cost: string | null;
+    calculated_total_cost: string | null;
+    latency: string | null;
+};
+export type Pricing = {
+    id: string;
+    model_name: string;
+    pricing_unit: Generated<PricingUnit>;
+    price: string;
+    currency: Generated<string>;
+    token_type: TokenType;
+};
+export type Project = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    name: string;
+    cloud_config: unknown | null;
+};
+export type Prompt = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    created_by: string;
+    prompt: string;
+    name: string;
+    version: number;
+    is_active: boolean;
+    config: Generated<unknown>;
+};
+export type Score = {
+    id: string;
+    timestamp: Generated<Timestamp>;
+    name: string;
+    value: number;
+    comment: string | null;
+    trace_id: string;
+    observation_id: string | null;
+};
+export type Session = {
+    id: string;
+    session_token: string;
+    user_id: string;
+    expires: Timestamp;
+};
+export type Trace = {
+    id: string;
+    external_id: string | null;
+    timestamp: Generated<Timestamp>;
+    name: string | null;
+    user_id: string | null;
+    metadata: unknown | null;
+    release: string | null;
+    version: string | null;
+    project_id: string;
+    public: Generated<boolean>;
+    bookmarked: Generated<boolean>;
+    tags: Generated<string[]>;
+    input: unknown | null;
+    output: unknown | null;
+    session_id: string | null;
+};
+export type TraceSession = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    bookmarked: Generated<boolean>;
+    public: Generated<boolean>;
+};
+export type User = {
+    id: string;
+    name: string | null;
+    email: string | null;
+    email_verified: Timestamp | null;
+    password: string | null;
+    image: string | null;
+    admin: Generated<boolean>;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    feature_flags: Generated<string[]>;
+};
+export type VerificationToken = {
+    identifier: string;
+    token: string;
+    expires: Timestamp;
+};
+export type DB = {
+    Account: Account;
+    api_keys: ApiKey;
+    audit_logs: AuditLog;
+    cron_jobs: CronJobs;
+    dataset_items: DatasetItem;
+    dataset_run_items: DatasetRunItems;
+    dataset_runs: DatasetRuns;
+    datasets: Dataset;
+    events: Events;
+    Example: Example;
+    membership_invitations: MembershipInvitation;
+    memberships: Membership;
+    models: Model;
+    observations: Observation;
+    observations_view: ObservationView;
+    pricings: Pricing;
+    projects: Project;
+    prompts: Prompt;
+    scores: Score;
+    Session: Session;
+    trace_sessions: TraceSession;
+    traces: Trace;
+    users: User;
+    verification_tokens: VerificationToken;
+};

prisma/migrations/20240219162415_add_prompt_config/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "prompts" ADD COLUMN     "config" JSONB NOT NULL DEFAULT '{}';

prisma/schema.prisma

@@ -21,6 +21,17 @@ generator erd {
     output       = "database.svg"
 }
 
+generator kysely {
+    provider = "prisma-kysely"
+
+    // Optionally provide a destination directory for the generated file
+    // and a filename of your choice
+    // output = "../src/db"
+    // fileName = "types.ts"
+    // Optionally generate runtime enums to a separate file
+    // enumFileName = "enums.ts"
+}
+
 model Example {
     id        String   @id @default(cuid())
     createdAt DateTime @default(now()) @map("created_at")
@@ -421,6 +432,7 @@ model Prompt {
     name        String
     version     Int
     isActive    Boolean       @map("is_active")
+    config      Json          @default("{}")
     Observation Observation[]
 
     @@unique([projectId, name, version])

prisma/seed.ts

@@ -176,7 +176,10 @@ async function main() {
         projectId: project2.id,
         createdBy: "user-1",
         prompt: "Prompt 4 version 1 content with {{variable}}",
-        name: "Prompt 4 with variable",
+        name: "Prompt 4 with variable and config",
+        config: {
+          temperature: 0.7,
+        },
         version: 1,
         isActive: false,
       },
@@ -185,7 +188,11 @@ async function main() {
         projectId: project2.id,
         createdBy: "user-1",
         prompt: "Prompt 4 version 2 content with {{variable}}",
-        name: "Prompt 4 with variable",
+        name: "Prompt 4 with variable and config",
+        config: {
+          temperature: 0.7,
+          topP: 0.9,
+        },
         version: 2,
         isActive: true,
       },
@@ -194,7 +201,12 @@ async function main() {
         projectId: project2.id,
         createdBy: "user-1",
         prompt: "Prompt 4 version 3 content with {{variable}}",
-        name: "Prompt 4 with variable",
+        name: "Prompt 4 with variable and config",
+        config: {
+          temperature: 0.7,
+          topP: 0.9,
+          frequencyPenalty: 0.5,
+        },
         version: 3,
         isActive: false,
       },
@@ -208,13 +220,14 @@ async function main() {
           createdBy: version.createdBy,
           prompt: version.prompt,
           name: version.name,
+          config: version.config,
           version: version.version,
           isActive: version.isActive,
         },
       });
       promptIds.push(version.id);
     }
-    const promptName = "Prompt with Longer Name";
+    const promptName = "Prompt with many versions";
     const projectId = project2.id;
     const createdBy = "user-1";
 
@@ -267,6 +280,7 @@ async function main() {
           ] as string,
           metadata: {
             user: `user-${i}@langfuse.com`,
+            more: "1,2,3;4?6",
           },
           tags: tags as string[],
           project: {

src/__tests__/prompts.servertest.ts

@@ -18,6 +18,9 @@ describe("/api/public/prompts API Endpoint", () => {
         prompt: "prompt",
         isActive: true,
         version: 1,
+        config: {
+          temperature: 0.1,
+        },
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
         },
@@ -43,6 +46,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
+    expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
   });
 
   it("should fetch active prompt only if no prompt version is given", async () => {
@@ -55,6 +59,9 @@ describe("/api/public/prompts API Endpoint", () => {
         prompt: "prompt",
         isActive: false,
         version: 1,
+        config: {
+          temperature: 0.1,
+        },
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
         },
@@ -82,6 +89,9 @@ describe("/api/public/prompts API Endpoint", () => {
         prompt: "prompt-one",
         isActive: false,
         version: 1,
+        config: {
+          temperature: 0.1,
+        },
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
         },
@@ -96,6 +106,9 @@ describe("/api/public/prompts API Endpoint", () => {
         prompt: "prompt",
         isActive: true,
         version: 2,
+        config: {
+          temperature: 0.2,
+        },
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
         },
@@ -121,6 +134,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(false);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
+    expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
   });
 
   it("should fetch active prompt when multiple exist", async () => {
@@ -134,6 +148,9 @@ describe("/api/public/prompts API Endpoint", () => {
         prompt: "prompt",
         isActive: false,
         version: 1,
+        config: {
+          temperature: 0.1,
+        },
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
         },
@@ -148,6 +165,9 @@ describe("/api/public/prompts API Endpoint", () => {
         prompt: "prompt",
         isActive: true,
         version: 2,
+        config: {
+          temperature: 0.2,
+        },
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
         },
@@ -173,6 +193,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.version).toBe(2);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("user-1");
+    expect(fetchedObservations.body.config).toEqual({ temperature: 0.2 });
   });
 
   it("should create and fetch a prompt", async () => {
@@ -181,6 +202,9 @@ describe("/api/public/prompts API Endpoint", () => {
       prompt: "prompt",
       isActive: true,
       projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      config: {
+        temperature: 0.1,
+      },
     });
 
     const fetchedObservations = await makeAPICall(
@@ -200,6 +224,7 @@ describe("/api/public/prompts API Endpoint", () => {
     expect(fetchedObservations.body.version).toBe(1);
     expect(fetchedObservations.body.isActive).toBe(true);
     expect(fetchedObservations.body.createdBy).toBe("API");
+    expect(fetchedObservations.body.config).toEqual({ temperature: 0.1 });
   });
 
   it("should relate generation to prompt", async () => {
@@ -327,6 +352,34 @@ describe("/api/public/prompts API Endpoint", () => {
 
     expect(dbGeneration).toBeNull();
   });
+
+  it("should create empty object if no config is provided", async () => {
+    await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      prompt: "prompt",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    const fetchedObservations = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=1",
+      undefined,
+    );
+
+    expect(fetchedObservations.status).toBe(200);
+
+    if (!isPrompt(fetchedObservations.body)) {
+      throw new Error("Expected body to be an array of observations");
+    }
+
+    expect(fetchedObservations.body.name).toBe("prompt-name");
+    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.version).toBe(1);
+    expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.createdBy).toBe("API");
+    expect(fetchedObservations.body.config).toEqual({});
+  });
 });
 
 const isPrompt = (x: unknown): x is Prompt => {
@@ -339,6 +392,7 @@ const isPrompt = (x: unknown): x is Prompt => {
     typeof prompt.prompt === "string" &&
     typeof prompt.isActive === "boolean" &&
     typeof prompt.projectId === "string" &&
-    typeof prompt.createdBy === "string"
+    typeof prompt.createdBy === "string" &&
+    typeof prompt.config === "object"
   );
 };

src/__tests__/traces.servertest.ts

@@ -4,7 +4,7 @@ import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { prisma } from "@/src/server/db";
 import { v4 as uuidv4 } from "uuid";
 
-interface TraceAPIResponse {
+interface GetTracesAPIResponse {
   data: Array<{
     id: string;
     [key: string]: unknown;
@@ -116,7 +116,7 @@ describe("/api/public/traces API Endpoint", () => {
     });
 
     // multiple tags
-    const traces = await makeAPICall<TraceAPIResponse>(
+    const traces = await makeAPICall<GetTracesAPIResponse>(
       "GET",
       "/api/public/traces?tags=tag-2&tags=tag-3",
     );
@@ -125,7 +125,7 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds).toEqual(["trace-3", "trace-1"]);
 
     // single tag
-    const traces2 = await makeAPICall<TraceAPIResponse>(
+    const traces2 = await makeAPICall<GetTracesAPIResponse>(
       "GET",
       "/api/public/traces?tags=tag-1",
     );
@@ -134,7 +134,7 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds2).toEqual(["trace-2", "trace-1"]);
 
     // wrong tag
-    const traces3 = await makeAPICall<TraceAPIResponse>(
+    const traces3 = await makeAPICall<GetTracesAPIResponse>(
       "GET",
       "/api/public/traces?tags=tag-10",
     );
@@ -143,7 +143,7 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds3).toEqual([]);
 
     // no tag
-    const traces4 = await makeAPICall<TraceAPIResponse>(
+    const traces4 = await makeAPICall<GetTracesAPIResponse>(
       "GET",
       "/api/public/traces?tags=",
     );
@@ -152,7 +152,7 @@ describe("/api/public/traces API Endpoint", () => {
     expect(traceIds4).toEqual(["trace-3", "trace-2", "trace-1"]);
   });
 
-  it("should handle totalCost and latency correctly", async () => {
+  it("should handle metrics correctly on GET traces and GET trace", async () => {
     await pruneDatabase();
 
     // Create a trace with some observations that have costs and latencies
@@ -182,8 +182,9 @@ describe("/api/public/traces API Endpoint", () => {
       endTime: "2021-01-01T00:20:00.000Z",
     });
 
+    // GET traces
     // Retrieve the trace with totalCost and latency
-    const traces = await makeAPICall<TraceAPIResponse>(
+    const traces = await makeAPICall<GetTracesAPIResponse>(
       "GET",
       `/api/public/traces`,
     );
@@ -193,5 +194,22 @@ describe("/api/public/traces API Endpoint", () => {
     // Check if the totalCost and latency are calculated correctly
     expect(traceData.totalCost).toBeCloseTo(15.75); // Sum of costs
     expect(traceData.latency).toBeCloseTo(1200); // Difference in seconds between min startTime and max endTime
+    expect(traceData.id).toBe(traceId);
+    expect(traceData.htmlPath).toContain(`/traces/${traceId}`);
+    expect(traceData.htmlPath).toContain(`/project/`); // do not know the projectId
+
+    // GET trace
+    // Retrieve the trace with total
+    const trace = await makeAPICall<{
+      id: string;
+      totalCost: number;
+      htmlPath: string;
+    }>("GET", `/api/public/traces/${traceId}`);
+    console.log(trace.body);
+    expect(trace.body.totalCost).toBeCloseTo(15.75);
+    expect(trace.body.id).toBe(traceId);
+    expect(trace.body.id).toBe(traceId);
+    expect(trace.body.htmlPath).toContain(`/traces/${traceId}`);
+    expect(trace.body.htmlPath).toContain(`/project/`); // do not know the projectId
   });
 });

src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.6.0";
+export const VERSION = "v2.7.0";

src/features/dashboard/server/dashboard-router.ts

@@ -1,7 +1,3 @@
-import {
-  dateTimeAggregationOptions,
-  dateTimeAggregationSettings,
-} from "@/src/features/dashboard/lib/timeseries-aggregation";
 import { z } from "zod";
 
 import {
@@ -25,72 +21,4 @@ export const dashboardRouter = createTRPCRouter({
     .query(async ({ input, ctx }) => {
       return await executeQuery(ctx.prisma, input.projectId, input);
     }),
-
-  scores: protectedProjectProcedure
-    .input(
-      z.object({
-        projectId: z.string(),
-        agg: z.enum(dateTimeAggregationOptions),
-      }),
-    )
-    .query(async ({ input, ctx }) => {
-      // queryRawUnsafe to add input.agg to the WHERE clause
-      const output = await ctx.prisma.$queryRawUnsafe<
-        {
-          date_trunc: Date;
-          values: {
-            [key: string]: number;
-          } | null;
-        }[]
-      >(`
-      WITH timeseries AS (
-        SELECT
-          date_trunc('${
-            dateTimeAggregationSettings[input.agg].date_trunc
-          }', dt) as date_trunc
-        FROM generate_series(
-          NOW() - INTERVAL '${input.agg}', NOW(), INTERVAL '1 minute'
-        ) as dt
-        WHERE dt > NOW() - INTERVAL '${input.agg}'
-        GROUP BY 1
-      ),
-      metrics AS (
-        SELECT 
-          date_trunc('${
-            dateTimeAggregationSettings[input.agg].date_trunc
-          }', scores.timestamp) as date_trunc,
-          scores.name as metric_name,
-          AVG(value) as avg_value
-        FROM scores
-        LEFT JOIN traces ON scores.trace_id = traces.id
-        WHERE scores.timestamp > NOW() - INTERVAL '${input.agg}'
-        AND traces.project_id = '${input.projectId}'
-        GROUP BY 1,2
-      ),
-      json_metrics AS (
-        SELECT
-          date_trunc,
-          jsonb_object_agg(metric_name, avg_value) as values
-        FROM metrics
-        GROUP BY 1
-      )
-      SELECT
-        timeseries.date_trunc,
-        json_metrics.values as values
-      FROM timeseries
-      LEFT JOIN json_metrics ON timeseries.date_trunc = json_metrics.date_trunc
-      ORDER BY 1
-      `);
-
-      return output.map((row) => ({
-        ...row,
-        values: row.values
-          ? Object.entries(row.values).map(([label, value]) => ({
-              label: "avg_" + label,
-              value,
-            }))
-          : [],
-        ts: row.date_trunc.getTime(),
-      }));
-    }),
 });

src/features/datasets/server/dataset-router.ts

@@ -7,6 +7,7 @@ import {
 import { type DatasetRuns, Prisma, type Dataset } from "@prisma/client";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { DB } from "@/src/server/db";
 
 export const datasetRouter = createTRPCRouter({
   allDatasets: protectedProjectProcedure
@@ -16,7 +17,30 @@ export const datasetRouter = createTRPCRouter({
       }),
     )
     .query(async ({ input, ctx }) => {
-      return ctx.prisma.$queryRaw<
+      const query = DB.selectFrom("datasets")
+        .leftJoin("dataset_items", "datasets.id", "dataset_items.dataset_id")
+        .leftJoin("dataset_runs", "datasets.id", "dataset_runs.dataset_id")
+        .select(({ eb }) => [
+          "datasets.id",
+          "datasets.name",
+          "datasets.created_at as createdAt",
+          "datasets.updated_at as updatedAt",
+          eb.fn.count("dataset_items.id").distinct().as("countDatasetItems"),
+          eb.fn.count("dataset_runs.id").distinct().as("countDatasetRuns"),
+          eb.fn.max("dataset_runs.created_at").as("lastRunAt"),
+        ])
+        .where("datasets.project_id", "=", input.projectId)
+        .groupBy([
+          "datasets.id",
+          "datasets.name",
+          "datasets.created_at",
+          "datasets.updated_at",
+        ])
+        .orderBy("datasets.created_at", "desc");
+
+      const compiledQuery = query.compile();
+
+      return await ctx.prisma.$queryRawUnsafe<
         Array<
           Dataset & {
             countDatasetItems: number;
@@ -24,22 +48,7 @@ export const datasetRouter = createTRPCRouter({
             lastRunAt: Date | null;
           }
         >
-      >(Prisma.sql`
-        SELECT
-          d.id,
-          d.name,
-          d.created_at "createdAt",
-          d.updated_at "updatedAt",
-          count(distinct di.id)::int "countDatasetItems",
-          count(distinct dr.id)::int "countDatasetRuns",
-          max(dr.created_at) "lastRunAt"
-        FROM datasets d
-        LEFT JOIN dataset_items di ON di.dataset_id = d.id
-        LEFT JOIN dataset_runs dr ON dr.dataset_id = d.id
-        WHERE d.project_id = ${input.projectId}
-        GROUP BY 1,2,3,4
-        ORDER BY d.created_at DESC
-      `);
+      >(compiledQuery.sql, ...compiledQuery.parameters);
     }),
   byId: protectedProjectProcedure
     .input(

src/features/filters/hooks/useFilterState.ts

@@ -19,15 +19,15 @@ const CommaArrayParam = {
       value.map((f) => {
         const stringified = `${f.column};${f.type};${
           f.type === "numberObject" || f.type === "stringObject" ? f.key : ""
-        };${f.operator};${
+        };${f.operator};${encodeURIComponent(
           f.type === "datetime"
             ? f.value.toISOString()
             : f.type === "stringOptions"
               ? f.value.join("|")
               : f.type === "arrayOptions"
                 ? f.value.join("|")
-                : f.value
-        }`;
+                : f.value,
+        )}`;
         // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
         if (DEBUG_QUERY_STATE) console.log("stringified", stringified);
         return stringified;
@@ -43,20 +43,21 @@ const CommaArrayParam = {
         // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
         if (DEBUG_QUERY_STATE)
           console.log("values", [column, type, key, operator, value]);
+        const decodedValue = value ? decodeURIComponent(value) : undefined;
         const parsedValue =
-          value === undefined || type === undefined
+          decodedValue === undefined || type === undefined
             ? undefined
             : type === "datetime"
-              ? new Date(value)
+              ? new Date(decodedValue)
               : type === "number" || type === "numberObject"
-                ? Number(value)
+                ? Number(decodedValue)
                 : type === "stringOptions"
-                  ? value.split("|")
+                  ? decodedValue.split("|")
                   : type === "arrayOptions"
-                    ? value.split("|")
+                    ? decodedValue.split("|")
                     : type === "boolean"
-                      ? value === "true"
-                      : value;
+                      ? decodedValue === "true"
+                      : decodedValue;
         // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
         if (DEBUG_QUERY_STATE) console.log("parsedValue", parsedValue);
         const parsed = singleFilter.safeParse({

src/features/prompts/components/new-prompt-button.tsx

@@ -15,6 +15,7 @@ import {
   FormControl,
   FormMessage,
   Form,
+  FormDescription,
 } from "@/src/components/ui/form";
 import { api } from "@/src/utils/api";
 import { zodResolver } from "@hookform/resolvers/zod";
@@ -29,6 +30,8 @@ import { Badge } from "@/src/components/ui/badge";
 import router from "next/router";
 import { AutoComplete } from "@/src/features/prompts/components/auto-complete";
 import { type AutoCompleteOption } from "@/src/features/prompts/components/auto-complete";
+import JsonView from "react18-json-view";
+import { jsonSchema } from "@/src/utils/zod";
 
 export const CreatePromptDialog = (props: {
   projectId: string;
@@ -36,6 +39,7 @@ export const CreatePromptDialog = (props: {
   promptName?: string;
   promptText?: string;
   subtitle?: string;
+  promptConfig?: z.infer<typeof jsonSchema>;
   children?: React.ReactNode;
 }) => {
   const [open, setOpen] = useState(false);
@@ -48,7 +52,7 @@ export const CreatePromptDialog = (props: {
   return (
     <Dialog open={hasAccess && open} onOpenChange={setOpen}>
       <DialogTrigger asChild>{props.children}</DialogTrigger>
-      <DialogContent className="sm:max-w-3xl">
+      <DialogContent className="max-h-screen overflow-auto sm:max-w-3xl">
         <DialogHeader>
           <DialogTitle className="mb-5">
             {props.title}
@@ -61,6 +65,7 @@ export const CreatePromptDialog = (props: {
           projectId={props.projectId}
           promptName={props.promptName}
           promptText={props.promptText}
+          promptConfig={props.promptConfig}
           onFormSuccess={() => setOpen(false)}
         />
       </DialogContent>
@@ -87,6 +92,20 @@ const formSchema = z.object({
   isActive: z.boolean({
     required_error: "Enter whether the prompt should go live",
   }),
+  // string as we keep the state in string to avoid recursive zod parsing issues
+  config: z.string().refine(
+    (value) => {
+      try {
+        JSON.parse(value);
+        return true;
+      } catch (e) {
+        return false;
+      }
+    },
+    {
+      message: "Config needs to be valid JSON",
+    },
+  ),
 });
 
 export const NewPromptForm = (props: {
@@ -94,6 +113,7 @@ export const NewPromptForm = (props: {
   onFormSuccess?: () => void;
   promptName?: string;
   promptText?: string;
+  promptConfig?: z.infer<typeof jsonSchema>;
 }) => {
   const [formError, setFormError] = useState<string | null>(null);
 
@@ -105,6 +125,7 @@ export const NewPromptForm = (props: {
       isActive: false,
       name: props.promptName ?? "",
       prompt: props.promptText ?? "",
+      config: props.promptConfig ? JSON.stringify(props.promptConfig) : "{}",
     },
   });
 
@@ -148,6 +169,9 @@ export const NewPromptForm = (props: {
         name: values.name,
         prompt: values.prompt,
         isActive: values.isActive,
+        // we keep the config in state as string. need to convert it to JSON before sending it to the API
+        // zod parsing necessary to align with TRPC schema
+        config: jsonSchema.parse(JSON.parse(values.config)),
       })
       .then((newPrompt) => {
         props.onFormSuccess?.();
@@ -229,6 +253,30 @@ export const NewPromptForm = (props: {
             </>
           )}
         />
+        <FormField
+          control={form.control}
+          name="config"
+          render={({ field }) => (
+            <FormItem>
+              <FormLabel>Config</FormLabel>
+              <JsonView
+                // need to convert string in state to JSON for the JSONView component
+                src={jsonSchema.parse(JSON.parse(field.value))}
+                onEdit={(edit) => {
+                  // need to put string back into the state
+                  field.onChange(JSON.stringify(edit.src));
+                }}
+                editable
+                className="rounded-md border border-gray-200 p-2 text-sm"
+              />
+              <FormDescription>
+                Track configs for LLM API calls such as function definitions or
+                LLM parameters.
+              </FormDescription>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
         <FormField
           control={form.control}
           name="isActive"

src/features/prompts/components/prompt-detail.tsx

@@ -14,7 +14,9 @@ import { PromotePrompt } from "@/src/features/prompts/components/promote-prompt"
 import { ScrollArea } from "@radix-ui/react-scroll-area";
 import { useQueryParam, NumberParam } from "use-query-params";
 import router from "next/router";
+import { JSONView } from "@/src/components/ui/code";
 import { DeletePromptVersion } from "@/src/features/prompts/components/delete-prompt-version";
+import { jsonSchema } from "@/src/utils/zod";
 
 export type PromptDetailProps = {
   projectId: string;
@@ -73,6 +75,7 @@ export const PromptDetail = (props: PromptDetailProps) => {
                   subtitle="We do not update prompts, instead we create a new version of the prompt."
                   promptName={prompt.name}
                   promptText={prompt.prompt}
+                  promptConfig={jsonSchema.parse(prompt.config)}
                 >
                   <Button variant="outline" size="icon">
                     <Pencil className="h-5 w-5" />
@@ -96,7 +99,7 @@ export const PromptDetail = (props: PromptDetailProps) => {
         </div>
         <div className="col-span-2 md:h-full">
           <CodeView content={prompt.prompt} title="Prompt" />
-          <div className="mx-auto mt-5 w-full rounded-lg border text-base leading-7 text-gray-700">
+          <div className="mx-auto mt-5 w-full rounded-lg border text-base leading-7">
             <div className="border-b px-3 py-1 text-xs font-medium">
               Variables
             </div>
@@ -112,6 +115,10 @@ export const PromptDetail = (props: PromptDetailProps) => {
               )}
             </div>
           </div>
+
+          {prompt.config && JSON.stringify(prompt.config) !== "{}" && (
+            <JSONView className="mt-5" json={prompt.config} title="Config" />
+          )}
         </div>
         <div className="flex h-screen flex-col">
           <div className="text-m px-3 font-medium">

src/features/prompts/server/prompt-router.ts

@@ -6,6 +6,7 @@ import {
 } from "@/src/server/api/trpc";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type Prompt, type PrismaClient } from "@prisma/client";
+import { jsonSchema } from "@/src/utils/zod";
 import { auditLog } from "@/src/features/audit-logs/auditLog";
 
 export const CreatePrompt = z.object({
@@ -13,6 +14,7 @@ export const CreatePrompt = z.object({
   name: z.string(),
   isActive: z.boolean(),
   prompt: z.string(),
+  config: jsonSchema,
 });
 
 export const promptRouter = createTRPCRouter({
@@ -77,6 +79,7 @@ export const promptRouter = createTRPCRouter({
           prompt: input.prompt,
           isActive: input.isActive,
           createdBy: ctx.session.user.id,
+          config: jsonSchema.parse(input.config),
           prisma: ctx.prisma,
         });
 
@@ -315,6 +318,7 @@ export const createPrompt = async ({
   prompt,
   isActive = true,
   createdBy,
+  config,
   prisma,
 }: {
   projectId: string;
@@ -322,6 +326,7 @@ export const createPrompt = async ({
   prompt: string;
   isActive?: boolean;
   createdBy: string;
+  config: z.infer<typeof jsonSchema>;
   prisma: PrismaClient;
 }) => {
   const latestPrompt = await prisma.prompt.findFirst({
@@ -350,6 +355,7 @@ export const createPrompt = async ({
         isActive: isActive,
         project: { connect: { id: projectId } },
         createdBy: createdBy,
+        config: jsonSchema.parse(config),
       },
     }),
   ];

src/pages/api/public/prompts.ts

@@ -2,6 +2,7 @@ import { createPrompt } from "@/src/features/prompts/server/prompt-router";
 import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
 import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
 import { prisma } from "@/src/server/db";
+import { jsonSchema } from "@/src/utils/zod";
 import { type NextApiRequest, type NextApiResponse } from "next";
 import { z } from "zod";
 
@@ -14,6 +15,7 @@ const PromptCreateSchema = z.object({
   name: z.string(),
   prompt: z.string(),
   isActive: z.boolean(),
+  config: jsonSchema.nullable().default({}),
 });
 
 export default async function handler(
@@ -56,7 +58,7 @@ export default async function handler(
           name: searchParams.name,
           version: searchParams.version ?? undefined,
           // if no version is given, we take the latest active prompt
-          // if no prompt is active, there will no prompt be available
+          // if no prompt is active, there will be no prompt available
           isActive: !searchParams.version ? true : undefined,
         },
       });
@@ -111,8 +113,10 @@ export default async function handler(
         prompt: input.prompt,
         isActive: input.isActive,
         createdBy: "API",
+        config: input.config ?? {},
         prisma: prisma,
       });
+      console.log("created prompt", prompt);
       return res.status(200).json(prompt);
     } catch (error: unknown) {
       console.error(error);

src/pages/api/public/traces/[traceId].ts

@@ -67,9 +67,17 @@ export default async function handler(
       },
     });
 
-    return res
-      .status(200)
-      .json({ ...trace, observations: observations.map(mapUsageOutput) });
+    const outObservations = observations.map(mapUsageOutput);
+
+    return res.status(200).json({
+      ...trace,
+      htmlPath: `/project/${authCheck.scope.projectId}/traces/${traceId}`,
+      totalCost: outObservations.reduce(
+        (acc, obs) => acc + (obs.calculatedTotalCost ?? 0),
+        0,
+      ),
+      observations: outObservations,
+    });
   } catch (error: unknown) {
     console.error(error);
     const errorMessage =

src/server/api/trpc.ts

@@ -18,7 +18,7 @@ import { type CreateNextContextOptions } from "@trpc/server/adapters/next";
 import { type Session } from "next-auth";
 
 import { getServerAuthSession } from "@/src/server/auth";
-import { prisma } from "@/src/server/db";
+import { DB, prisma } from "@/src/server/db";
 
 import * as z from "zod";
 
@@ -40,6 +40,7 @@ export const createInnerTRPCContext = (opts: CreateContextOptions) => {
   return {
     session: opts.session,
     prisma,
+    DB,
   };
 };
 

src/server/db.ts

@@ -1,6 +1,14 @@
 import { PrismaClient } from "@prisma/client";
 
 import { env } from "@/src/env.mjs";
+import {
+  DummyDriver,
+  Kysely,
+  PostgresAdapter,
+  PostgresIntrospector,
+  PostgresQueryCompiler,
+} from "kysely";
+import {  type DB as Database } from "@/prisma/generated/types";
 
 const globalForPrisma = globalThis as unknown as {
   prisma: PrismaClient | undefined;
@@ -13,4 +21,11 @@ export const prisma =
       env.NODE_ENV === "development" ? ["query", "error", "warn"] : ["error"],
   });
 
-if (env.NODE_ENV !== "production") globalForPrisma.prisma = prisma;
+export const DB = new Kysely<Database>({
+  dialect: {
+    createAdapter: () => new PostgresAdapter(),
+    createDriver: () => new DummyDriver(),
+    createIntrospector: (db) => new PostgresIntrospector(db),
+    createQueryCompiler: () => new PostgresQueryCompiler(),
+  },
+});