chore: release v2.47.5

---------

Co-authored-by: Marc Klingen <git@marcklingen.com>

.env.dev.example

@@ -47,5 +47,5 @@ REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
 LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
-# openssl rand -base64 32 used only here
+# openssl rand -hex 32 used only here
 ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 

.env.prod.example

@@ -58,6 +58,11 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_AUTH0_CLIENT_SECRET=
 # AUTH_AUTH0_ISSUER=
 # AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false
+# AUTH_COGNITO_CLIENT_ID=
+# AUTH_COGNITO_CLIENT_SECRET=
+# AUTH_COGNITO_ISSUER=
+# AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+
 
 # Transactional email, optional
 # Defines the email address to use as the from address.

.github/workflows/porter_stack_worker.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - main
+name: Deploy to worker
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "3959"
+        PORTER_DEPLOYMENT_TARGET_ID: d2bb23e4-1d77-48f2-a113-383d076959b3
+        PORTER_HOST: https://dashboard.getporter.dev
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: worker
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_3959 }}

README.md

@@ -106,22 +106,26 @@ See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Lan
 
 ### Integrations
 
-| Integration                                                      | Supports      | Description                                                 |
-| ---------------------------------------------------------------- | ------------- | ----------------------------------------------------------- |
-| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
-| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python, JS/TS | Automated instrumentation of OpenAI SDK.                    |
-| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.                    |
-| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python        | Automated instrumentation via LlamaIndex callback system.   |
-| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.       |
+| Integration                                              | Supports                   | Description                                                                                                                                      |
+| -------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [SDK](/docs/sdk)                                         | Python, JS/TS              | Manual instrumentation using the SDKs for full flexibility.                                                                                      |
+| [OpenAI](/docs/integrations/openai)                      | Python, JS/TS              | Automated instrumentation using drop-in replacement of OpenAI SDK.                                                                               |
+| [Langchain](/docs/integrations/langchain)                | Python, JS/TS              | Automated instrumentation by passing callback handler to Langchain application.                                                                  |
+| [LlamaIndex](/docs/integrations/llama-index/get-started) | Python                     | Automated instrumentation via LlamaIndex callback system.                                                                                        |
+| [Haystack](/docs/integrations/haystack)                  | Python                     | Automated instrumentation via Haystack content tracing system.                                                                                   |
+| [LiteLLM](/docs/integrations/litellm)                    | Python, JS/TS (proxy only) | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
+| [API](/docs/api)                                         |                            | Directly call the public API. OpenAPI spec available.                                                                                            |
 
-External projects/packages that integrate with Langfuse:
+Packages that integrate with Langfuse:
 
-| Name                                                            | Description                                                                                                                                      |
-| --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [LiteLLM](https://langfuse.com/docs/integrations/litellm)       | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
-| [Flowise](https://langfuse.com/docs/integrations/flowise)       | JS/TS no-code builder for customized LLM flows.                                                                                                  |
-| [Langflow](https://langfuse.com/docs/integrations/langflow)     | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows.                          |
-| [Superagent](https://langfuse.com/docs/integrations/superagent) | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                                               |
+| Name                                                       | Description                                                                                                             |
+| ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| [Instructor](/docs/integrations/instructor)                | Library to get structured LLM outputs (JSON, Pydantic)                                                                  |
+| [Mirascope](/docs/integrations/mirascope)                  | Python toolkit for building LLM applications.                                                                           |
+| [AI SDK by Vercel](/docs/sdk/typescript/example-vercel-ai) | Typescript SDK that makes streaming LLM outputs super easy.                                                             |
+| [Flowise](/docs/integrations/flowise)                      | JS/TS no-code builder for customized LLM flows.                                                                         |
+| [Langflow](/docs/integrations/langflow)                    | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
+| [Superagent](/docs/integrations/superagent)                | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                      |
 
 ## Questions and feedback
 

careers

@@ -3,9 +3,9 @@ Join us in scaling Langfuse in Berlin, Germany. We are an open source company, w
 
 _Open Roles_
 
-- Backend Engineer, 70-110k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
-- Product Engineer, 70-110k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
-- Developer Advocate, 60-100k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+- Backend Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
+- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
 
 _More Info_
 

ee/src/sso/index.ts

@@ -2,10 +2,11 @@ import { type Provider } from "next-auth/providers/index";
 import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
 import OktaProvider from "next-auth/providers/okta";
+import CognitoProvider from "next-auth/providers/cognito";
 import Auth0Provider from "next-auth/providers/auth0";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { isEeAvailable } from "..";
-import { prisma } from "@langfuse/shared/src/db";
+import { type SsoConfig, prisma } from "@langfuse/shared/src/db";
 import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import { type NextApiRequest, type NextApiResponse } from "next";
@@ -25,14 +26,36 @@ let cachedSsoConfigs: {
 async function getSsoConfigs(): Promise<SsoProviderSchema[]> {
   if (!isEeAvailable) return [];
   const CACHE_TTL = 60 * 1000; // 1 minute
+  const DB_MAX_WAIT = 2000; // 2 seconds
+  const DB_TIMEOUT = 3000; // 3 seconds
 
   // Set/refresh the cache if it's empty or expired
   if (
     cachedSsoConfigs === null ||
     Date.now() - cachedSsoConfigs.timestamp > CACHE_TTL
   ) {
+    // findMany with custom timeout via $transaction
+    let dbConfigs: SsoConfig[] = [];
+    try {
+      dbConfigs = await prisma.$transaction(
+        async (prisma) => prisma.ssoConfig.findMany(),
+        {
+          maxWait: DB_MAX_WAIT,
+          timeout: DB_TIMEOUT,
+        }
+      );
+    } catch (e) {
+      // cache empty array to prevent repeated DB calls on error
+      cachedSsoConfigs = {
+        data: [],
+        timestamp: Date.now(),
+      };
+
+      // caught and logged in the caller
+      throw e;
+    }
+
     // transform into zod object
-    const dbConfigs = await prisma.ssoConfig.findMany();
     const parsedSsoConfigs = dbConfigs
       .map((v) => {
         try {
@@ -144,6 +167,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "cognito")
+    return CognitoProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line no-unused-vars

ee/src/sso/types.ts

@@ -64,11 +64,24 @@ export const AzureAdProviderSchema = base.extend({
     .nullish(),
 });
 
+export const CognitoProviderSchema = base.extend({
+  authProvider: z.literal("cognito"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
@@ -76,6 +89,7 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   Auth0ProviderSchema,
   OktaProviderSchema,
   AzureAdProviderSchema,
+  CognitoProviderSchema,
 ]);
 
 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

fern/apis/server/definition/commons.yml

@@ -209,8 +209,8 @@ types:
       - ARCHIVED
   ScoreSource:
     enum:
+      - ANNOTATION
       - API
-      - REVIEW
       - EVAL
 
 errors:

fern/apis/server/definition/prompts.yml

@@ -70,14 +70,24 @@ types:
       name: string
       prompt: list<ChatMessage>
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   CreateTextPromptRequest:
     properties:
       name: string
       prompt: string
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   Prompt:
     union:
@@ -89,7 +99,12 @@ types:
       name: string
       version: integer
       config: unknown
-      labels: list<string>
+      labels:
+        type: list<string>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: list<string>
+        docs: List of tags. Used to filter via UI and API. The same across versions of a prompt.
 
   ChatMessage:
     properties:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.45.0",
+  "version": "2.47.5",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/prisma/generated/types.ts

@@ -25,11 +25,16 @@ export const ObservationLevel = {
 } as const;
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
+    ANNOTATION: "ANNOTATION",
     API: "API",
-    REVIEW: "REVIEW",
     EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
+export const ScoreDataType = {
+    CATEGORICAL: "CATEGORICAL",
+    NUMERIC: "NUMERIC"
+} as const;
+export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
 export const PricingUnit = {
     PER_1000_TOKENS: "PER_1000_TOKENS",
     PER_1000_CHARS: "PER_1000_CHARS"
@@ -246,6 +251,7 @@ export type Observation = {
     status_message: string | null;
     version: string | null;
     created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
     model: string | null;
     internal_model: string | null;
     modelParameters: unknown | null;
@@ -351,6 +357,24 @@ export type Score = {
     comment: string | null;
     trace_id: string;
     observation_id: string | null;
+    config_id: string | null;
+    string_value: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    data_type: Generated<ScoreDataType>;
+};
+export type ScoreConfig = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    data_type: ScoreDataType;
+    is_archived: Generated<boolean>;
+    min_value: number | null;
+    max_value: number | null;
+    categories: unknown | null;
+    description: string | null;
 };
 export type Session = {
     id: string;
@@ -450,6 +474,7 @@ export type DB = {
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
+    score_configs: ScoreConfig;
     scores: Score;
     Session: Session;
     sso_configs: SsoConfig;

packages/shared/prisma/migrations/20240523142425_score_config_add_table/migration.sql

@@ -0,0 +1,42 @@
+-- CreateEnum
+CREATE TYPE "ScoreDataType" AS ENUM ('CATEGORICAL', 'NUMERIC');
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "config_id" TEXT,
+ADD COLUMN     "data_type" "ScoreDataType" NOT NULL DEFAULT 'NUMERIC',
+ADD COLUMN     "string_value" TEXT;
+
+-- CreateTable
+CREATE TABLE "score_configs" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "data_type" "ScoreDataType" NOT NULL,
+    "is_archived" BOOLEAN NOT NULL DEFAULT false,
+    "min_value" DOUBLE PRECISION,
+    "max_value" DOUBLE PRECISION,
+    "categories" JSONB,
+    "description" TEXT,
+
+    CONSTRAINT "score_configs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "score_configs_data_type_idx" ON "score_configs"("data_type");
+
+-- CreateIndex
+CREATE INDEX "score_configs_is_archived_idx" ON "score_configs"("is_archived");
+
+-- CreateIndex
+CREATE INDEX "score_configs_project_id_idx" ON "score_configs"("project_id");
+
+-- CreateIndex
+CREATE INDEX "score_configs_categories_idx" ON "score_configs"("categories");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "score_configs_id_project_id_key" ON "score_configs"("id", "project_id");
+
+-- AddForeignKey
+ALTER TABLE "score_configs" ADD CONSTRAINT "score_configs_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240523142524_scores_add_config_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_config_id_idx" ON "scores"("config_id");

packages/shared/prisma/migrations/20240523142610_scores_add_fk_scores_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_config_id_fkey" FOREIGN KEY ("config_id") REFERENCES "score_configs"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240524154058_scores_source_enum_add_annotation/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'ANNOTATION';

packages/shared/prisma/migrations/20240524156058_scores_source_backfill_annotation_for_review/migration.sql

@@ -0,0 +1,4 @@
+-- Backfill the scores source for 'REVIEW' to be 'ANNOTATION'
+UPDATE "scores"
+SET "source" = 'ANNOTATION'::"ScoreSource"
+WHERE "source" = 'REVIEW'::"ScoreSource";

packages/shared/prisma/migrations/20240524165931_scores_source_enum_drop_review/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - The values [REVIEW] on the enum `ScoreSource` will be removed. If these variants are still used in the database, this will fail.
+
+*/
+-- AlterEnum
+BEGIN;
+CREATE TYPE "ScoreSource_new" AS ENUM ('ANNOTATION', 'API', 'EVAL');
+ALTER TABLE "scores" ALTER COLUMN "source" TYPE "ScoreSource_new" USING ("source"::text::"ScoreSource_new");
+ALTER TYPE "ScoreSource" RENAME TO "ScoreSource_old";
+ALTER TYPE "ScoreSource_new" RENAME TO "ScoreSource";
+DROP TYPE "ScoreSource_old";
+COMMIT;

packages/shared/prisma/migrations/20240524190433_job_executions_add_fk_index_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_configuration_id_idx" ON "job_executions"("job_configuration_id");

packages/shared/prisma/migrations/20240524190434_job_executions_add_fk_index_score_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_output_score_id_idx" ON "job_executions"("job_output_score_id");

packages/shared/prisma/migrations/20240524190435_job_executions_add_fk_index_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_input_trace_id_idx" ON "job_executions"("job_input_trace_id");

packages/shared/prisma/migrations/20240524190436_job_executions_index_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_created_at_idx" ON "job_executions"("created_at");

packages/shared/prisma/migrations/20240528214726_add_cursor_new_columns_observations/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240528214727_add_cursor_new_columns_scores/migration.sql

@@ -0,0 +1,4 @@
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240528214728_add_cursor_index_01/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "audit_logs_updated_at_idx" ON "audit_logs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_02/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_created_at_idx" ON "dataset_items"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_03/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_updated_at_idx" ON "dataset_items"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_04/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_created_at_idx" ON "dataset_run_items"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_05/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_updated_at_idx" ON "dataset_run_items"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_06/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_created_at_idx" ON "dataset_runs"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_07/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_updated_at_idx" ON "dataset_runs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_08/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_created_at_idx" ON "datasets"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_09/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_updated_at_idx" ON "datasets"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_10/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_updated_at_idx" ON "job_executions"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_11/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_updated_at_idx" ON "observations"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_12/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_created_at_idx" ON "prompts"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_13/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_updated_at_idx" ON "prompts"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_14/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_created_at_idx" ON "score_configs"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_15/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_updated_at_idx" ON "score_configs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_16/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_created_at_idx" ON "scores"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_17/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_updated_at_idx" ON "scores"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_18/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "trace_sessions_updated_at_idx" ON "trace_sessions"("updated_at");

packages/shared/prisma/migrations/20240530212419_model_price_anthropic_via_google_vertex/migration.sql

@@ -0,0 +1,7 @@
+-- Google Vertex uses @ to separate model name and version
+
+UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-haiku(-|@)?20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-opus(-|@)?20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-sonnet(-|@)?20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';

packages/shared/prisma/schema.prisma

@@ -114,6 +114,7 @@ model Project {
     LlmApiKeys         LlmApiKeys[]
     PosthogIntegration PosthogIntegration[]
     Score              Score[]
+    scoreConfig        ScoreConfig[]
 
     @@map("projects")
 }
@@ -205,6 +206,7 @@ model TraceSession {
     @@id([id, projectId])
     @@index([projectId])
     @@index([createdAt])
+    @@index([updatedAt])
     @@map("trace_sessions")
 }
 
@@ -292,6 +294,7 @@ model Observation {
     statusMessage       String?          @map("status_message")
     version             String?
     createdAt           DateTime         @default(now()) @map("created_at")
+    updatedAt           DateTime         @default(now()) @updatedAt @map("updated_at")
 
     // GENERATION ONLY
     model               String?
@@ -323,6 +326,7 @@ model Observation {
     @@index([type])
     @@index(startTime)
     @@index(createdAt)
+    @@index(updatedAt)
     @@index(projectId)
     @@index(parentObservationId)
     @@index(model)
@@ -403,25 +407,64 @@ model Score {
     comment       String?
     traceId       String         @map("trace_id")
     observationId String?        @map("observation_id")
+    configId      String?        @map("config_id")
+    stringValue   String?        @map("string_value")
+    createdAt     DateTime       @default(now()) @map("created_at")
+    updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
+    dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
     JobExecution  JobExecution[]
+    scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
 
     @@unique([id, projectId]) // used for upserts via prisma
     @@index(timestamp)
     @@index([value])
     @@index([projectId])
     @@index([authorUserId])
+    @@index([configId])
     @@index([traceId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([source])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("scores")
 }
 
 enum ScoreSource {
+    ANNOTATION
     API
-    REVIEW
     EVAL
 }
 
+model ScoreConfig {
+    id          String        @id @default(cuid())
+    createdAt   DateTime      @default(now()) @map("created_at")
+    updatedAt   DateTime      @default(now()) @updatedAt @map("updated_at")
+    projectId   String        @map("project_id")
+    project     Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    name        String
+    dataType    ScoreDataType @map("data_type")
+    isArchived  Boolean       @default(false) @map("is_archived")
+    minValue    Float?        @map("min_value")
+    maxValue    Float?        @map("max_value")
+    categories  Json?         @map("categories")
+    description String?
+    score       Score[]
+
+    @@unique([id, projectId]) // used for upserts via prisma
+    @@index([dataType])
+    @@index([isArchived])
+    @@index([projectId])
+    @@index([categories])
+    @@index([createdAt])
+    @@index([updatedAt])
+    @@map("score_configs")
+}
+
+enum ScoreDataType {
+    CATEGORICAL
+    NUMERIC
+}
+
 enum PricingUnit {
     PER_1000_TOKENS
     PER_1000_CHARS
@@ -468,6 +511,8 @@ model Dataset {
 
     @@unique([projectId, name])
     @@index([projectId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("datasets")
 }
 
@@ -489,6 +534,8 @@ model DatasetItem {
 
     @@index([sourceObservationId], type: Hash)
     @@index([datasetId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("dataset_items")
 }
 
@@ -510,6 +557,8 @@ model DatasetRuns {
 
     @@unique([datasetId, name])
     @@index([datasetId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("dataset_runs")
 }
 
@@ -530,6 +579,8 @@ model DatasetRunItems {
     @@index([datasetItemId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([traceId])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("dataset_run_items")
 }
 
@@ -572,6 +623,8 @@ model Prompt {
     @@index([projectId, name, version])
     @@index([projectId, id])
     @@index([projectId])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("prompts")
 }
@@ -619,6 +672,7 @@ model AuditLog {
 
     @@index([projectId])
     @@index([createdAt])
+    @@index([updatedAt])
     @@map("audit_logs")
 }
 
@@ -712,6 +766,11 @@ model JobExecution {
     @@index([projectId, status])
     @@index([projectId, id])
     @@index([projectId])
+    @@index([jobConfigurationId])
+    @@index([jobOutputScoreId])
+    @@index([jobInputTraceId])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("job_executions")
 }
 

packages/shared/prisma/seed.ts

@@ -12,6 +12,7 @@ import { chunk } from "lodash";
 import { v4 } from "uuid";
 import { ModelUsageUnit } from "../src";
 import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
+import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -165,6 +166,24 @@ async function main() {
 
     await uploadObjects(traces, observations, scores, sessions, events);
 
+    // If openai key is in environment, add it to the projects LLM API keys
+    const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+
+    if (OPENAI_API_KEY) {
+      await prisma.llmApiKeys.create({
+        data: {
+          projectId: project1.id,
+          secretKey: encrypt(OPENAI_API_KEY),
+          displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
+          provider: "openai",
+        },
+      });
+    } else {
+      console.warn(
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key."
+      );
+    }
+
     // add eval objects
     const evalTemplate = await prisma.evalTemplate.upsert({
       where: {
@@ -500,7 +519,7 @@ function createObjects(
               name: "manual-score",
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
-              source: ScoreSource.REVIEW,
+              source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
             },

pnpm-lock.yaml

@@ -222,6 +222,9 @@ importers:
       '@codemirror/lang-json':
         specifier: ^6.0.1
         version: 6.0.1
+      '@codemirror/lint':
+        specifier: ^6.8.0
+        version: 6.8.0
       '@headlessui/react':
         specifier: 1.7.18
         version: 1.7.18(react-dom@18.2.0)(react@18.2.0)
@@ -380,7 +383,7 @@ importers:
         version: 4.22.1(@codemirror/language@6.10.1)(@codemirror/state@6.4.1)(@codemirror/view@6.26.3)
       '@uiw/react-codemirror':
         specifier: ^4.21.25
-        version: 4.21.25(@babel/runtime@7.24.5)(@codemirror/autocomplete@6.16.0)(@codemirror/language@6.10.1)(@codemirror/lint@6.7.1)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.26.3)(codemirror@6.0.1)(react-dom@18.2.0)(react@18.2.0)
+        version: 4.21.25(@babel/runtime@7.24.5)(@codemirror/autocomplete@6.16.0)(@codemirror/language@6.10.1)(@codemirror/lint@6.8.0)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.26.3)(codemirror@6.0.1)(react-dom@18.2.0)(react@18.2.0)
       ai:
         specifier: ^3.0.23
         version: 3.0.23(react@18.2.0)(solid-js@1.8.17)(svelte@4.2.17)(vue@3.4.27)(zod@3.22.4)
@@ -466,8 +469,8 @@ importers:
         specifier: ^7.51.5
         version: 7.51.5(react@18.2.0)
       react-icons:
-        specifier: ^5.0.1
-        version: 5.0.1(react@18.2.0)
+        specifier: ^5.2.1
+        version: 5.2.1(react@18.2.0)
       react-responsive:
         specifier: ^10.0.0
         version: 10.0.0(react@18.2.0)
@@ -2084,6 +2087,14 @@ packages:
       crelt: 1.0.6
     dev: false
 
+  /@codemirror/lint@6.8.0:
+    resolution: {integrity: sha512-lsFofvaw0lnPRJlQylNsC4IRt/1lI4OD/yYslrSGVndOJfStc58v+8p9dgGiD90ktOfL7OhBWns1ZETYgz0EJA==}
+    dependencies:
+      '@codemirror/state': 6.4.1
+      '@codemirror/view': 6.26.3
+      crelt: 1.0.6
+    dev: false
+
   /@codemirror/search@6.5.6:
     resolution: {integrity: sha512-rpMgcsh7o0GuCDUXKPvww+muLA1pDJaFrpq/CCHtpQJYz8xopu4D1hPcKRoDD0YlF8gZaqTNIRa4VRBWyhyy7Q==}
     dependencies:
@@ -7111,7 +7122,7 @@ packages:
       eslint-visitor-keys: 3.4.3
     dev: true
 
-  /@uiw/codemirror-extensions-basic-setup@4.21.25(@codemirror/autocomplete@6.16.0)(@codemirror/commands@6.3.3)(@codemirror/language@6.10.1)(@codemirror/lint@6.7.1)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/view@6.26.3):
+  /@uiw/codemirror-extensions-basic-setup@4.21.25(@codemirror/autocomplete@6.16.0)(@codemirror/commands@6.3.3)(@codemirror/language@6.10.1)(@codemirror/lint@6.8.0)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/view@6.26.3):
     resolution: {integrity: sha512-eeUKlmEE8aSoSgelS8OR2elcPGntpRo669XinAqPCLa0eKorT2B0d3ts+AE+njAeGk744tiyAEbHb2n+6OQmJw==}
     peerDependencies:
       '@codemirror/autocomplete': '>=6.0.0'
@@ -7125,7 +7136,7 @@ packages:
       '@codemirror/autocomplete': 6.16.0(@codemirror/language@6.10.1)(@codemirror/state@6.4.1)(@codemirror/view@6.26.3)(@lezer/common@1.2.1)
       '@codemirror/commands': 6.3.3
       '@codemirror/language': 6.10.1
-      '@codemirror/lint': 6.7.1
+      '@codemirror/lint': 6.8.0
       '@codemirror/search': 6.5.6
       '@codemirror/state': 6.4.1
       '@codemirror/view': 6.26.3
@@ -7175,7 +7186,7 @@ packages:
       '@codemirror/view': 6.26.3
     dev: false
 
-  /@uiw/react-codemirror@4.21.25(@babel/runtime@7.24.5)(@codemirror/autocomplete@6.16.0)(@codemirror/language@6.10.1)(@codemirror/lint@6.7.1)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.26.3)(codemirror@6.0.1)(react-dom@18.2.0)(react@18.2.0):
+  /@uiw/react-codemirror@4.21.25(@babel/runtime@7.24.5)(@codemirror/autocomplete@6.16.0)(@codemirror/language@6.10.1)(@codemirror/lint@6.8.0)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.26.3)(codemirror@6.0.1)(react-dom@18.2.0)(react@18.2.0):
     resolution: {integrity: sha512-mBrCoiffQ+hbTqV1JoixFEcH7BHXkS3PjTyNH7dE8Gzf3GSBRazhtSM5HrAFIiQ5FIRGFs8Gznc4UAdhtevMmw==}
     peerDependencies:
       '@babel/runtime': '>=7.11.0'
@@ -7191,7 +7202,7 @@ packages:
       '@codemirror/state': 6.4.1
       '@codemirror/theme-one-dark': 6.1.2
       '@codemirror/view': 6.26.3
-      '@uiw/codemirror-extensions-basic-setup': 4.21.25(@codemirror/autocomplete@6.16.0)(@codemirror/commands@6.3.3)(@codemirror/language@6.10.1)(@codemirror/lint@6.7.1)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/view@6.26.3)
+      '@uiw/codemirror-extensions-basic-setup': 4.21.25(@codemirror/autocomplete@6.16.0)(@codemirror/commands@6.3.3)(@codemirror/language@6.10.1)(@codemirror/lint@6.8.0)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/view@6.26.3)
       codemirror: 6.0.1(@lezer/common@1.2.1)
       react: 18.2.0
       react-dom: 18.2.0(react@18.2.0)
@@ -15018,8 +15029,8 @@ packages:
       react: 18.2.0
     dev: false
 
-  /react-icons@5.0.1(react@18.2.0):
-    resolution: {integrity: sha512-WqLZJ4bLzlhmsvme6iFdgO8gfZP17rfjYEJ2m9RsZjZ+cc4k1hTzknEz63YS1MeT50kVzoa1Nz36f4BEx+Wigw==}
+  /react-icons@5.2.1(react@18.2.0):
+    resolution: {integrity: sha512-zdbW5GstTzXaVKvGSyTaBalt7HSfuK5ovrzlpyiWHAFXndXTdd/1hdDHI4xBM1Mn7YriT6aqESucFl9kEXzrdw==}
     peerDependencies:
       react: '*'
     dependencies:

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.45.0",
+  "version": "2.47.5",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -25,6 +25,7 @@
     "@aws-sdk/lib-storage": "^3.568.0",
     "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@codemirror/lang-json": "^6.0.1",
+    "@codemirror/lint": "^6.8.0",
     "@headlessui/react": "1.7.18",
     "@headlessui/tailwindcss": "0.2.0",
     "@heroicons/react": "^2.1.3",
@@ -106,7 +107,7 @@
     "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
     "react-hook-form": "^7.51.5",
-    "react-icons": "^5.0.1",
+    "react-icons": "^5.2.1",
     "react-responsive": "^10.0.0",
     "react18-json-view": "^0.2.8-canary.6",
     "sonner": "^1.4.41",

web/public/generated/api/openapi.yml

@@ -1825,8 +1825,8 @@ components:
       title: ScoreSource
       type: string
       enum:
+        - ANNOTATION
         - API
-        - REVIEW
         - EVAL
     CreateDatasetItemRequest:
       title: CreateDatasetItemRequest
@@ -2654,6 +2654,13 @@ components:
           items:
             type: string
           nullable: true
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of tags to apply to all versions of this prompt.
       required:
         - name
         - prompt
@@ -2672,6 +2679,13 @@ components:
           items:
             type: string
           nullable: true
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          nullable: true
+          description: List of tags to apply to all versions of this prompt.
       required:
         - name
         - prompt
@@ -2713,11 +2727,20 @@ components:
           type: array
           items:
             type: string
+          description: List of deployment labels of this prompt version.
+        tags:
+          type: array
+          items:
+            type: string
+          description: >-
+            List of tags. Used to filter via UI and API. The same across
+            versions of a prompt.
       required:
         - name
         - version
         - config
         - labels
+        - tags
     ChatMessage:
       title: ChatMessage
       type: object

web/public/generated/postman/collection.json

@@ -699,7 +699,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ]\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ]\n}",
               "options": {
                 "raw": {
                   "language": "json"

web/sentry.server.config.ts

@@ -18,7 +18,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
         samplingContext.request.url &&
         samplingContext.request.url.includes("api/trpc")
       ) {
-        return 0.3;
+        return 0.1;
       }
       if (
         samplingContext.request &&
@@ -27,12 +27,12 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
         samplingContext.transactionContext.status !== "ok" &&
         samplingContext.transactionContext.status !== "unauthenticated"
       ) {
-        return 1;
+        return 0.1;
       }
-      return 0.1;
+      return 0.01;
     },
 
-    profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate
+    profilesSampleRate: 0.1,
     integrations: [
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),

web/src/__tests__/prompts.v2.servertest.ts

@@ -680,6 +680,71 @@ describe("/api/public/v2/prompts API Endpoint", () => {
       expect(fetchedPrompt.body.createdBy).toBe("API");
       expect(fetchedPrompt.body.config).toEqual({});
     });
+
+    it("should update tags across versions", async () => {
+      const promptName = "prompt-name" + nanoid();
+
+      const createPromptVersion = async (tags?: string[]) => {
+        await makeAPICall("POST", baseURI, {
+          name: promptName,
+          prompt: "This is a test prompt",
+          type: PromptType.Text,
+          ...(tags !== undefined && { tags: tags }),
+        });
+      };
+
+      const fetchPromptVersion = async (version: number) => {
+        const fetchedPrompt = await makeAPICall(
+          "GET",
+          `${baseURI}/${promptName}?version=${version}`,
+          undefined,
+        );
+        expect(fetchedPrompt.status).toBe(200);
+        if (!isPrompt(fetchedPrompt.body)) {
+          throw new Error("Expected body to be a prompt");
+        }
+        return fetchedPrompt.body;
+      };
+
+      // Create version 1 with ["tag"]
+      await createPromptVersion(["tag"]);
+      let fetchedPrompt1 = await fetchPromptVersion(1);
+      expect(fetchedPrompt1.tags).toEqual(["tag"]);
+      expect(fetchedPrompt1.version).toBe(1);
+
+      // Create version 2 with no tags provided (should use tags from version 1)
+      await createPromptVersion();
+      let fetchedPrompt2 = await fetchPromptVersion(2);
+      expect(fetchedPrompt2.tags).toEqual(["tag"]);
+      expect(fetchedPrompt2.version).toBe(2);
+
+      // Create version 3 with ["tag1", "tag2", "tag3"] (should update tags across versions)
+      await createPromptVersion(["tag1", "tag2", "tag3"]);
+      fetchedPrompt1 = await fetchPromptVersion(1);
+      fetchedPrompt2 = await fetchPromptVersion(2);
+      let fetchedPrompt3 = await fetchPromptVersion(3);
+      expect(fetchedPrompt1.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt1.version).toBe(1);
+      expect(fetchedPrompt2.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt2.version).toBe(2);
+      expect(fetchedPrompt3.tags).toEqual(["tag1", "tag2", "tag3"]);
+      expect(fetchedPrompt3.version).toBe(3);
+
+      // remove tags
+      await createPromptVersion([]);
+      fetchedPrompt1 = await fetchPromptVersion(1);
+      fetchedPrompt2 = await fetchPromptVersion(2);
+      fetchedPrompt3 = await fetchPromptVersion(3);
+      let fetchedPrompt4 = await fetchPromptVersion(4);
+      expect(fetchedPrompt1.tags).toEqual([]);
+      expect(fetchedPrompt1.version).toBe(1);
+      expect(fetchedPrompt2.tags).toEqual([]);
+      expect(fetchedPrompt2.version).toBe(2);
+      expect(fetchedPrompt3.tags).toEqual([]);
+      expect(fetchedPrompt3.version).toBe(3);
+      expect(fetchedPrompt4.tags).toEqual([]);
+      expect(fetchedPrompt4.version).toBe(4);
+    });
   });
 
   describe("when fetching a prompt list", () => {

web/src/__tests__/scores.servertest.ts

@@ -489,16 +489,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
         {
           id: scoreId_2,
           name: scoreName,
           value: 50.5,
         },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
       ]);
     });
     it("test operator >=", async () => {
@@ -514,16 +514,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
         {
           id: scoreId_3,
           name: scoreName,
           value: 100.8,
         },
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
       ]);
     });
     it("test operator !=", async () => {
@@ -539,16 +539,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
         {
           id: scoreId_3,
           name: scoreName,
           value: 100.8,
         },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
       ]);
     });
     it("test operator =", async () => {

web/src/app/api/chatCompletion/route.ts

@@ -0,0 +1,6 @@
+import chatCompletionHandler from "@/src/ee/features/playground/server/chatCompletionHandler";
+
+export const dynamic = "force-dynamic";
+export const maxDuration = 60;
+
+export const POST = chatCompletionHandler;

web/src/components/ModelParameters/LLMApiKeyComponent.tsx

@@ -0,0 +1,77 @@
+import Link from "next/link";
+
+import { Label } from "@/src/components/ui/label";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { api } from "@/src/utils/api";
+import { supportedModels, type UIModelParams } from "@langfuse/shared";
+import { ArrowTopRightIcon } from "@radix-ui/react-icons";
+
+export const LLMApiKeyComponent = (p: {
+  projectId: string;
+  modelParams: UIModelParams;
+}) => {
+  const hasAccess = useHasAccess({
+    projectId: p.projectId,
+    scope: "llmApiKeys:read",
+  });
+
+  if (!hasAccess) {
+    return (
+      <div>
+        <Label className="text-xs font-semibold">API key</Label>
+        <p className="text-sm text-muted-foreground">
+          LLM API Key only visible to Owner and Admin roles.
+        </p>
+      </div>
+    );
+  }
+
+  const apiKeys = api.llmApiKey.all.useQuery({
+    projectId: p.projectId,
+  });
+
+  if (apiKeys.isLoading) {
+    return (
+      <div>
+        <Label className="text-xs font-semibold">API key</Label>
+        <p className="text-sm text-muted-foreground">Loading...</p>
+      </div>
+    );
+  }
+
+  const model = p.modelParams.model.value;
+  const modelProvider = Object.entries(supportedModels).find((providerData) =>
+    (providerData[1] as any as string[]).includes(model),
+  )?.[0];
+
+  const apiKey = apiKeys.data?.data.find((k) => k.provider === modelProvider);
+
+  return (
+    <div className="space-y-2 text-xs">
+      <Label className="text-xs font-semibold">API key</Label>
+      <div>
+        {apiKey ? (
+          <span className="mr-2 rounded-sm bg-input p-1 text-xs">
+            {apiKey.displaySecretKey}
+          </span>
+        ) : undefined}
+      </div>
+      {/* Custom form message to include a link to the already existing prompt */}
+      {!apiKey ? (
+        <div className="flex flex-col font-medium text-destructive">
+          {`No LLM API key found for provider ${modelProvider}.`}
+
+          <Link
+            href={`/project/${p.projectId}/settings`}
+            className="flex flex-row"
+          >
+            Create a new LLM API key here. <ArrowTopRightIcon />
+          </Link>
+        </div>
+      ) : undefined}
+      <p className="text-muted-foreground">
+        The LLM API key is used for each execution and will incur costs.
+      </p>
+    </div>
+  );
+};

web/src/components/ModelParameters/index.tsx

@@ -8,6 +8,7 @@ import {
 } from "@/src/components/ui/select";
 import { Slider } from "@/src/components/ui/slider";
 import { Switch } from "@/src/components/ui/switch";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { cn } from "@/src/utils/tailwind";
 import {
   ModelProvider,
@@ -15,6 +16,8 @@ import {
   type UIModelParams,
 } from "@langfuse/shared";
 
+import { LLMApiKeyComponent } from "./LLMApiKeyComponent";
+
 export type ModelParamsContext = {
   modelParams: UIModelParams;
   availableModels?: UIModelParams[];
@@ -33,10 +36,14 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
   setModelParamEnabled,
   formDisabled = false,
 }) => {
+  const projectId = useProjectIdFromURL();
+
+  if (!projectId) return null;
+
   return (
     <div className="flex flex-col space-y-4">
       <p className="font-semibold">Model</p>
-      <div className="space-y-6">
+      <div className="space-y-4">
         <ModelParamsSelect
           title="Provider"
           modelParamsKey="provider"
@@ -104,6 +111,7 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
           tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
           updateModelParam={updateModelParamValue}
         />
+        <LLMApiKeyComponent {...{ projectId, modelParams }} />
       </div>
     </div>
   );

web/src/components/Support.tsx

@@ -8,14 +8,7 @@ import {
   CardTitle,
 } from "@/src/components/ui/card";
 import { chatAvailable, openChat } from "@/src/features/support-chat/chat";
-import {
-  Book,
-  Calendar,
-  Github,
-  Mail,
-  MessageSquare,
-  Slack,
-} from "lucide-react";
+import { Book, Github, Mail, MessageSquare, Slack } from "lucide-react";
 import Link from "next/link";
 import { SiDiscord } from "react-icons/si";
 
@@ -30,7 +23,7 @@ const supportChannels = [
   },
   {
     icon: Github,
-    title: "GitHub",
+    title: "GitHub Issues",
     description:
       "Create an issue on Github to report bugs or request new features.",
     href: "https://github.com/langfuse/langfuse/issues/new/choose",
@@ -38,12 +31,11 @@ const supportChannels = [
     primary: true,
   },
   {
-    icon: SiDiscord,
-    title: "Discord",
-    description:
-      "Get support from community and maintainers. Follow announcements to stay up to date with new features.",
-    href: "https://langfuse.com/discord",
-    buttonText: "Join Discord",
+    icon: Github,
+    title: "GitHub Support",
+    description: "Create a support ticket via GitHub discussions.",
+    href: "https://github.com/orgs/langfuse/discussions/categories/support",
+    buttonText: "Submit question",
     primary: true,
   },
   {
@@ -57,17 +49,18 @@ const supportChannels = [
   {
     icon: MessageSquare,
     title: "Chat",
-    description: "Get quick support directly from the team.",
+    description: "Get support directly from the team.",
     onClick: () => openChat(),
     available: chatAvailable,
     buttonText: "Launch Chat",
   },
   {
-    icon: Calendar,
-    title: "Schedule call",
-    description: "Schedule a call with one of the founders.",
-    href: "https://cal.com/marc-kl/office-hours",
-    buttonText: "Schedule Call",
+    icon: SiDiscord,
+    title: "Discord",
+    description:
+      "Get support from community. Follow announcements to stay up to date with new features.",
+    href: "https://langfuse.com/discord",
+    buttonText: "Join Discord",
   },
   {
     icon: Slack,

web/src/components/json-editor.tsx

@@ -1,9 +1,11 @@
 import CodeMirror, { EditorView } from "@uiw/react-codemirror";
 import { githubLight } from "@uiw/codemirror-theme-github";
 import { tokyoNight } from "@uiw/codemirror-theme-tokyo-night";
-import { json } from "@codemirror/lang-json";
+import { json, jsonParseLinter } from "@codemirror/lang-json";
+import { linter } from "@codemirror/lint";
 import { useTheme } from "next-themes";
 import { cn } from "@/src/utils/tailwind";
+import { useState } from "react";
 
 // todo: add json linting
 
@@ -20,8 +22,13 @@ export function JsonEditor({
   lineWrapping?: boolean;
   className?: string;
 }) {
-  const { theme } = useTheme();
-  const codeMirrorTheme = theme === "dark" ? tokyoNight : githubLight;
+  const { resolvedTheme } = useTheme();
+  const codeMirrorTheme = resolvedTheme === "dark" ? tokyoNight : githubLight;
+  
+  // used to disable linter when field is empty
+  const [linterEnabled, setLinterEnabled] = useState<boolean>(
+    !!defaultValue && defaultValue !== "",
+  );
   return (
     <CodeMirror
       value={defaultValue}
@@ -30,9 +37,16 @@ export function JsonEditor({
         foldGutter: true,
       }}
       lang={"json"}
-      extensions={[json(), ...(lineWrapping ? [EditorView.lineWrapping] : [])]}
+      extensions={[
+        json(),
+        ...(linterEnabled ? [linter(jsonParseLinter())] : []),
+        ...(lineWrapping ? [EditorView.lineWrapping] : []),
+      ]}
       defaultValue={defaultValue}
-      onChange={onChange}
+      onChange={(c) => {
+        if (onChange) onChange(c);
+        setLinterEnabled(c !== "");
+      }}
       className={cn("overflow-hidden rounded-md border", className)}
       editable={editable}
     />

web/src/components/layouts/doc-popup.tsx

@@ -4,6 +4,7 @@ import {
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { cn } from "@/src/utils/tailwind";
 import { HelpCircle, Info } from "lucide-react";
 import Link from "next/link";
 
@@ -40,7 +41,10 @@ export default function DocPopup({
         }
       }}
     >
-      <HoverCardTrigger className="mx-1 cursor-pointer" asChild>
+      <HoverCardTrigger
+        className={cn("mx-1", href ? "cursor-pointer" : "cursor-default")}
+        asChild
+      >
         {href ? (
           <Link
             href={href}

web/src/components/star-toggle.tsx

@@ -7,6 +7,7 @@ import { cn } from "@/src/utils/tailwind";
 import { type RouterOutput, type RouterInput } from "@/src/utils/types";
 import { useState } from "react";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
 
 export function StarToggle({
   value,
@@ -76,7 +77,7 @@ export function StarTraceToggle({
     onError: (err, _newTodo, context) => {
       setIsLoading(false);
       // Rollback to the previous value if mutation fails
-      console.log("error", err);
+      trpcErrorToast(err);
       utils.traces.all.setData(tracesFilter, context?.prev);
     },
     onSettled: () => {
@@ -157,7 +158,7 @@ export function StarTraceDetailsToggle({
     },
     onError: (err, _newTodo, context) => {
       setIsLoading(false);
-      console.log("error", err);
+      trpcErrorToast(err);
       // Rollback to the previous value if mutation fails
       utils.traces.byId.setData({ traceId }, context?.prevData);
     },

web/src/components/table/use-cases/generations.tsx

@@ -21,11 +21,7 @@ import {
   withDefault,
 } from "use-query-params";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
-import {
-  formatIntervalSeconds,
-  intervalInSeconds,
-  utcDateOffsetByDays,
-} from "@/src/utils/dates";
+import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import {
@@ -119,7 +115,7 @@ export default function GenerationsTable({
         type: "datetime",
         operator: ">",
         value: utcDateOffsetByDays(
-          session.data?.environment.defaultTableDateTimeOffset ?? -14,
+          session.data?.environment.defaultTableDateTimeOffset ?? -7,
         ),
       },
     ],

web/src/components/table/use-cases/scores.tsx

@@ -12,8 +12,10 @@ import {
   scoresTableColsWithOptions,
 } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
+import { utcDateOffsetByDays } from "@/src/utils/dates";
 import type { RouterOutput, RouterInput } from "@/src/utils/types";
 import type { FilterState } from "@langfuse/shared";
+import { useSession } from "next-auth/react";
 import { useQueryParams, withDefault, NumberParam } from "use-query-params";
 
 export type ScoresTableRow = {
@@ -72,11 +74,20 @@ export default function ScoresTable({
     pageIndex: withDefault(NumberParam, 0),
     pageSize: withDefault(NumberParam, 50),
   });
-
+  const session = useSession();
   const [rowHeight, setRowHeight] = useRowHeightLocalStorage("scores", "s");
 
   const [userFilterState, setUserFilterState] = useQueryFilterState(
-    [],
+    [
+      {
+        column: "Timestamp",
+        type: "datetime",
+        operator: ">",
+        value: utcDateOffsetByDays(
+          session.data?.environment.defaultTableDateTimeOffset ?? -7,
+        ),
+      },
+    ],
     "scores",
   );
 

web/src/components/table/use-cases/sessions.tsx

@@ -55,7 +55,7 @@ export default function SessionsTable({
         type: "datetime",
         operator: ">",
         value: utcDateOffsetByDays(
-          session.data?.environment.defaultTableDateTimeOffset ?? -14,
+          session.data?.environment.defaultTableDateTimeOffset ?? -7,
         ),
       },
     ],

web/src/components/table/use-cases/traces.tsx

@@ -96,7 +96,7 @@ export default function TracesTable({
         type: "datetime",
         operator: ">",
         value: utcDateOffsetByDays(
-          session.data?.environment.defaultTableDateTimeOffset ?? -14,
+          session.data?.environment.defaultTableDateTimeOffset ?? -7,
         ),
       },
     ],
@@ -547,6 +547,7 @@ export default function TracesTable({
       header: "Version",
       enableHiding: true,
       enableSorting: true,
+      defaultHidden: true,
     },
     {
       accessorKey: "release",
@@ -554,6 +555,7 @@ export default function TracesTable({
       header: "Release",
       enableHiding: true,
       enableSorting: true,
+      defaultHidden: true,
     },
     {
       accessorKey: "tags",

web/src/components/trace/index.tsx

@@ -277,6 +277,7 @@ export function TracePage({ traceId }: { traceId: string }) {
             availableTags={allTags}
             traceId={trace.data.id}
             projectId={trace.data.projectId}
+            className="flex-wrap"
           />
         </div>
       </div>

web/src/components/ui/CodeJsonViewer.tsx

@@ -17,7 +17,7 @@ export function JSONView(props: {
 }) {
   // some users ingest stringified json nested in json, parse it
   const parsedJson = deepParseJson(props.json);
-  const { theme } = useTheme();
+  const { resolvedTheme } = useTheme();
   return (
     <div className={cn("rounded-md border", props.className)}>
       {props.title ? (
@@ -44,7 +44,7 @@ export function JSONView(props: {
           <React18JsonView
             src={parsedJson}
             theme="github"
-            dark={theme === "dark"}
+            dark={resolvedTheme === "dark"}
             collapseObjectsAfterLength={20}
             collapseStringsAfterLength={500}
             displaySize={"collapsed"}

web/src/components/ui/button.tsx

@@ -19,6 +19,8 @@ const buttonVariants = cva(
           "bg-secondary text-secondary-foreground hover:bg-secondary/80",
         ghost: "hover:bg-accent hover:text-accent-foreground",
         link: "text-primary underline-offset-4 hover:underline",
+        errorNotification:
+          "bg-destructive-foreground/90 text-destructive hover:bg-destructive-foreground/80",
       },
       size: {
         default: "h-10 px-4 py-2",

web/src/components/ui/command.tsx

@@ -42,7 +42,7 @@ const CommandInput = React.forwardRef<
   React.ComponentPropsWithoutRef<typeof CommandPrimitive.Input>
 >(({ className, ...props }, ref) => (
   <div
-    className="flex items-center rounded-lg border px-3"
+    className="flex items-center rounded-md border px-3"
     cmdk-input-wrapper=""
   >
     <Search className="mr-2 h-4 w-4 shrink-0 opacity-50" />

web/src/components/ui/table.tsx

@@ -74,7 +74,7 @@ const TableHead = React.forwardRef<
   <th
     ref={ref}
     className={cn(
-      "sticky top-0 z-10 border-b bg-background px-4 text-left align-middle font-medium text-muted-foreground [&:has([role=checkbox])]:pr-0",
+      "sticky top-0 z-10 rounded-t-md border-b bg-background px-4 text-left align-middle font-medium text-muted-foreground [&:has([role=checkbox])]:pr-0",
       className,
     )}
     {...props}

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.45.0";
+export const VERSION = "v2.47.5";

web/src/ee/features/evals/components/template-form.tsx

@@ -33,8 +33,6 @@ import {
   type ModelParams,
 } from "@langfuse/shared";
 import { PromptDescription } from "@/src/features/prompts/components/prompt-description";
-import Link from "next/dist/client/link";
-import { ArrowTopRightIcon } from "@radix-ui/react-icons";
 import {
   Select,
   SelectContent,
@@ -43,8 +41,6 @@ import {
   SelectValue,
 } from "@/src/components/ui/select";
 import { TEMPLATES } from "@/src/ee/features/evals/components/templates";
-import { Label } from "@/src/components/ui/label";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
 
@@ -439,10 +435,6 @@ export const InnerEvalTemplateForm = (props: {
               availableModels={[...evalLLMModels]}
               formDisabled={!props.isEditing}
             />
-            <LLMApiKeyComponent
-              projectId={props.projectId}
-              modelParams={modelParams}
-            />
           </div>
         </div>
 
@@ -465,78 +457,6 @@ export const InnerEvalTemplateForm = (props: {
   );
 };
 
-export const LLMApiKeyComponent = (p: {
-  projectId: string;
-  modelParams: UIModelParams;
-}) => {
-  const hasAccess = useHasAccess({
-    projectId: p.projectId,
-    scope: "llmApiKeys:read",
-  });
-
-  if (!hasAccess) {
-    return (
-      <div>
-        <Label>API key</Label>
-        <p className="text-sm text-muted-foreground">
-          LLM API Key only visible to Owner and Admin roles.
-        </p>
-      </div>
-    );
-  }
-
-  const apiKeys = api.llmApiKey.all.useQuery({
-    projectId: p.projectId,
-  });
-
-  if (apiKeys.isLoading) {
-    return (
-      <div>
-        <Label>API key</Label>
-        <p className="text-sm text-muted-foreground">Loading...</p>
-      </div>
-    );
-  }
-
-  const getModelProvider = (model: string) => {
-    return evalLLMModels.find((m) => m.model.value === model)?.provider.value;
-  };
-
-  const getApiKeyForModel = (model: string) => {
-    const modelProvider = getModelProvider(model);
-    return apiKeys.data?.data.find((k) => k.provider === modelProvider);
-  };
-
-  return (
-    <div>
-      <Label>API key</Label>
-      <div>
-        {getApiKeyForModel(p.modelParams.model.value) ? (
-          <span className="mr-2 rounded-sm bg-input p-1 text-xs">
-            {getApiKeyForModel(p.modelParams.model.value)?.displaySecretKey}
-          </span>
-        ) : undefined}
-      </div>
-      {/* Custom form message to include a link to the already existing prompt */}
-      {!getApiKeyForModel(p.modelParams.model.value) ? (
-        <div className="flex flex-col text-sm font-medium text-destructive">
-          {"No LLM API key found."}
-
-          <Link
-            href={`/project/${p.projectId}/settings`}
-            className="flex flex-row"
-          >
-            Create a new API key here. <ArrowTopRightIcon />
-          </Link>
-        </div>
-      ) : undefined}
-      <p className="text-sm text-muted-foreground">
-        The API key is used for each evaluation and will incur costs.
-      </p>
-    </div>
-  );
-};
-
 function getModelParamsWithEnabledFlag(
   evalPreFill?: EvalTemplateFormPreFill,
 ): UIModelParams {

web/src/ee/features/playground/page/components/SaveToPromptButton.tsx

@@ -49,8 +49,7 @@ export const SaveToPromptButton: React.FC = () => {
       )
       .data?.prompts.filter((prompt) => prompt.type === PromptType.Chat)
       .map((prompt) => ({
-        label:
-          prompt.name.slice(0, 20) + (prompt.name.length > 25 ? "..." : ""),
+        label: prompt.name,
         value: prompt.id,
       })) ?? [];
 
@@ -126,7 +125,9 @@ export const SaveToPromptButton: React.FC = () => {
                         : "opacity-0",
                     )}
                   />
-                  {promptName.label}
+                  <span className="overflow-hidden text-ellipsis whitespace-nowrap">
+                    {promptName.label}
+                  </span>
                 </CommandItem>
               ))}
             </CommandList>

web/src/ee/features/playground/page/context/index.tsx

@@ -14,6 +14,7 @@ import useCommandEnter from "@/src/ee/features/playground/page/hooks/useCommandE
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { extractVariables } from "@/src/utils/string";
 import {
   ChatMessageRole,
@@ -57,6 +58,7 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
   children,
 }) => {
   const capture = usePostHogClientCapture();
+  const projectId = useProjectIdFromURL();
   const { playgroundCache, setPlaygroundCache } = usePlaygroundCache();
   const [promptVariables, setPromptVariables] = useState<PromptVariable[]>([]);
   const [output, setOutput] = useState("");
@@ -172,6 +174,7 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
         }
 
         const completionStream = getChatCompletionStream(
+          projectId,
           finalMessages,
           modelParams,
         );
@@ -263,10 +266,17 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
 };
 
 async function* getChatCompletionStream(
+  projectId: string | undefined,
   messages: ChatMessageWithId[],
   modelParams: UIModelParams,
 ) {
+  if (!projectId) {
+    console.error("Project ID is not set");
+    return;
+  }
+
   const body = JSON.stringify({
+    projectId,
     messages,
     modelParams: getFinalModelParams(modelParams),
   });

web/src/ee/features/playground/page/playground.tsx

@@ -16,7 +16,7 @@ export default function Playground() {
           <div className="basis-[55%] ">
             <ModelParameters {...playgroundContext} />
           </div>
-          <div className="basis-[45%]  overflow-auto">
+          <div className="mt-4 basis-[45%] overflow-auto">
             <Variables />
           </div>
         </div>

web/src/ee/features/playground/server/authorizeRequest.ts

@@ -0,0 +1,26 @@
+import { getServerSession } from "next-auth";
+
+import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
+import { getAuthOptions } from "@/src/server/auth";
+import { isProjectMemberOrAdmin } from "@/src/server/utils/checkProjectMembershipOrAdmin";
+import { ApiError, ForbiddenError, UnauthorizedError } from "@langfuse/shared";
+
+export type AuthorizeRequestResult = {
+  userId: string;
+};
+
+export const authorizeRequestOrThrow = async (
+  projectId: string,
+): Promise<AuthorizeRequestResult> => {
+  if (!getIsCloudEnvironment())
+    throw new ApiError("This endpoint is available in Langfuse cloud only.");
+
+  const authOptions = await getAuthOptions();
+  const session = await getServerSession(authOptions);
+  if (!session?.user) throw new UnauthorizedError("Unauthenticated");
+
+  if (!isProjectMemberOrAdmin(session.user, projectId))
+    throw new ForbiddenError("User is not a member of this project");
+
+  return { userId: session.user.id };
+};

web/src/ee/features/playground/server/chatCompletionHandler.ts

@@ -1,75 +1,60 @@
 import { StreamingTextResponse } from "ai";
-import { getToken } from "next-auth/jwt";
-import { type NextRequest, NextResponse } from "next/server";
+import { NextResponse, type NextRequest } from "next/server";
 
-import { fetchLLMCompletion } from "@langfuse/shared";
+import {
+  BaseError,
+  ValidationError,
+  fetchLLMCompletion,
+} from "@langfuse/shared";
 
 import { PosthogCallbackHandler } from "./analytics/posthogCallback";
-import {
-  validateChatCompletionBody,
-  type ValidatedChatCompletionBody,
-} from "./validateChatCompletionBody";
-import { getCookieName } from "@/src/server/utils/cookies";
-import { env } from "@/src/env.mjs";
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
+import { authorizeRequestOrThrow } from "./authorizeRequest";
+import { validateChatCompletionBody } from "./validateChatCompletionBody";
+
+import { prisma } from "@langfuse/shared/src/db";
+import { decrypt } from "@langfuse/shared/encryption";
 
 export default async function chatCompletionHandler(req: NextRequest) {
-  if (!getIsCloudEnvironment()) {
-    return NextResponse.json(
-      { message: "This endpoint is available in Langfuse cloud only." },
-      { status: 501 },
-    );
-  }
-
-  const token = await getToken({
-    req,
-    cookieName: getCookieName("next-auth.session-token"),
-    secret: env.NEXTAUTH_SECRET,
-  });
-
-  if (!token || !token.sub)
-    // sub is the user id
-    return NextResponse.json({ message: "Unauthenticated" }, { status: 401 });
-
-  if (req.method !== "POST")
-    return NextResponse.json(
-      { message: "Method not allowed" },
-      { status: 405 },
-    );
-
-  let body: ValidatedChatCompletionBody;
-
   try {
-    body = validateChatCompletionBody(await req.json());
-  } catch (err) {
-    console.error(err);
+    const body = validateChatCompletionBody(await req.json());
+    const { userId } = await authorizeRequestOrThrow(body.projectId);
 
-    return NextResponse.json(
-      {
-        message: "Invalid request body",
-        error: err,
-      },
-      { status: 400 },
-    );
-  }
-
-  try {
     const { messages, modelParams } = body;
+
+    const LLMApiKey = await prisma.llmApiKeys.findFirst({
+      where: {
+        projectId: body.projectId,
+        provider: modelParams.provider,
+      },
+    });
+
+    if (!LLMApiKey)
+      throw new ValidationError(
+        `No ${modelParams.provider} API key found in project. Please add one in the project settings.`,
+      );
+
     const stream = await fetchLLMCompletion({
       messages,
       modelParams,
       streaming: true,
-      callbacks: [new PosthogCallbackHandler("playground", body, token.sub)],
-      apiKey:
-        modelParams.provider === "openai"
-          ? env.OPENAI_API_KEY
-          : env.ANTHROPIC_API_KEY,
+      callbacks: [new PosthogCallbackHandler("playground", body, userId)],
+      apiKey: decrypt(LLMApiKey.secretKey),
     });
 
     return new StreamingTextResponse(stream);
   } catch (err) {
     console.error(err);
 
+    if (err instanceof BaseError) {
+      return NextResponse.json(
+        {
+          error: err.name,
+          message: err.message,
+        },
+        { status: err.httpCode },
+      );
+    }
+
     if (err instanceof Error) {
       return NextResponse.json(
         {

web/src/ee/features/playground/server/validateChatCompletionBody.ts

@@ -32,6 +32,7 @@ const MessageSchema = z.object({
   id: z.string().optional(),
 });
 export const ChatCompletionBodySchema = z.object({
+  projectId: z.string(),
   messages: z.array(MessageSchema),
   modelParams: ModelParamsSchema,
 });

web/src/env.mjs

@@ -67,6 +67,10 @@ export const env = createEnv({
     AUTH_AUTH0_CLIENT_SECRET: z.string().optional(),
     AUTH_AUTH0_ISSUER: z.string().url().optional(),
     AUTH_AUTH0_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_COGNITO_CLIENT_ID: z.string().optional(),
+    AUTH_COGNITO_CLIENT_SECRET: z.string().optional(),
+    AUTH_COGNITO_ISSUER: z.string().url().optional(),
+    AUTH_COGNITO_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT: z.string().optional(),
     AUTH_DISABLE_USERNAME_PASSWORD: z.enum(["true", "false"]).optional(),
     AUTH_DISABLE_SIGNUP: z.enum(["true", "false"]).optional(),
@@ -167,6 +171,10 @@ export const env = createEnv({
     AUTH_AUTH0_ISSUER: process.env.AUTH_AUTH0_ISSUER,
     AUTH_AUTH0_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
+    AUTH_COGNITO_CLIENT_ID: process.env.AUTH_COGNITO_CLIENT_ID,
+    AUTH_COGNITO_CLIENT_SECRET: process.env.AUTH_COGNITO_CLIENT_SECRET,
+    AUTH_COGNITO_ISSUER: process.env.AUTH_COGNITO_ISSUER,
+    AUTH_COGNITO_ALLOW_ACCOUNT_LINKING: process.env.AUTH_COGNITO_ALLOW_ACCOUNT_LINKING,
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT:
       process.env.AUTH_DOMAINS_WITH_SSO_ENFORCEMENT,
     AUTH_DISABLE_USERNAME_PASSWORD: process.env.AUTH_DISABLE_USERNAME_PASSWORD,

web/src/features/datasets/server/dataset-router.ts

@@ -137,50 +137,6 @@ export const datasetRouter = createTRPCRouter({
           }
         >
       >(Prisma.sql`
-        WITH avg_scores_by_run_id AS (
-          SELECT
-            ri.dataset_run_id run_id,
-            s.name score_name,
-            AVG(s.value) AS average_score_value
-          FROM
-            dataset_run_items ri
-            JOIN scores s 
-              ON s.trace_id = ri.trace_id 
-              AND (ri.observation_id IS NULL OR s.observation_id = ri.observation_id) -- only include scores that are linked to the observation if observation is linked
-              AND s.project_id = ${input.projectId}
-            JOIN traces t ON t.id = s.trace_id
-          WHERE t.project_id = ${input.projectId}
-          GROUP BY
-            ri.dataset_run_id,
-            s.name
-          ORDER BY
-            1,
-            2
-        ),
-        json_avg_scores_by_run_id AS (
-          SELECT
-            run_id,
-            jsonb_object_agg(score_name,
-              average_score_value) AS scores
-          FROM
-            avg_scores_by_run_id
-          GROUP BY
-            run_id
-          ORDER BY
-            run_id
-        ),
-        latency_and_total_cost_by_run_id AS (
-          SELECT
-            ri.dataset_run_id run_id,
-            AVG(CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END)  AS "avgLatency",
-            AVG(COALESCE(o.calculated_total_cost, 0)) AS "avgTotalCost"
-          FROM
-            dataset_run_items ri
-            JOIN observations_view o ON o.id = ri.observation_id
-          WHERE o.project_id = ${input.projectId}
-          group by 1
-        )
-
         SELECT
           runs.id,
           runs.name,
@@ -188,28 +144,52 @@ export const datasetRouter = createTRPCRouter({
           runs.metadata,
           runs.created_at "createdAt",
           runs.updated_at "updatedAt",
-          COALESCE(avg_scores.scores, '[]'::jsonb) scores,
+          COALESCE(avg_scores.scores, '{}') scores,
           COALESCE(latency_and_total_cost."avgLatency", 0) "avgLatency",
           COALESCE(latency_and_total_cost."avgTotalCost", 0) "avgTotalCost",  
-          count(DISTINCT ri.id)::int "countRunItems"
+          COALESCE(run_items_count.count, 0)::int "countRunItems"
         FROM
           dataset_runs runs
           JOIN datasets ON datasets.id = runs.dataset_id
-          LEFT JOIN dataset_run_items ri ON ri.dataset_run_id = runs.id
-          LEFT JOIN json_avg_scores_by_run_id avg_scores ON avg_scores.run_id = runs.id
-          LEFT JOIN latency_and_total_cost_by_run_id latency_and_total_cost ON latency_and_total_cost.run_id = runs.id
-        WHERE runs.dataset_id = ${input.datasetId}
+          LEFT JOIN LATERAL (
+            SELECT
+              jsonb_object_agg(s.name, s.avg_value) AS scores
+            FROM (
+              SELECT
+                s.name,
+                AVG(s.value) AS avg_value
+              FROM
+                dataset_run_items ri
+                JOIN scores s 
+                  ON s.trace_id = ri.trace_id 
+                  AND (ri.observation_id IS NULL OR s.observation_id = ri.observation_id)
+                  AND s.project_id = ${input.projectId}
+                JOIN traces t ON t.id = s.trace_id
+              WHERE 
+                t.project_id = ${input.projectId}
+                AND ri.dataset_run_id = runs.id
+              GROUP BY s.name
+            ) s
+          ) avg_scores ON true
+          LEFT JOIN LATERAL (
+            SELECT
+              AVG(o.latency) AS "avgLatency",
+              AVG(COALESCE(o.calculated_total_cost, 0)) AS "avgTotalCost"
+            FROM
+              dataset_run_items ri
+              JOIN observations_view o ON o.id = ri.observation_id
+            WHERE 
+              o.project_id = ${input.projectId}
+              AND ri.dataset_run_id = runs.id
+          ) latency_and_total_cost ON true
+          LEFT JOIN LATERAL (
+            SELECT count(*) as count 
+            FROM dataset_run_items ri 
+            WHERE ri.dataset_run_id = runs.id
+          ) run_items_count ON true
+        WHERE 
+          runs.dataset_id = ${input.datasetId}
           AND datasets.project_id = ${input.projectId}
-        GROUP BY
-          1,
-          2,
-          3,
-          4,
-          5,
-          6,
-          7,
-          8,
-          9
         ORDER BY
           runs.created_at DESC
         LIMIT ${input.limit}
@@ -550,7 +530,6 @@ export const datasetRouter = createTRPCRouter({
           },
         },
         include: {
-          datasetItem: true,
           observation: {
             select: {
               id: true,

web/src/features/email/lib/project-invitation.ts

@@ -11,7 +11,7 @@ const langfuseUrls = {
   STAGING: "https://staging.langfuse.com",
 };
 
-const authUrl =
+const getAuthURL = () =>
   env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "US" ||
   env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "EU" ||
   env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
@@ -40,7 +40,7 @@ export const sendProjectInvitation = async (
         invitedByUserEmail: inviterEmail,
         projectName: projectName,
         recieverEmail: to,
-        inviteLink: authUrl,
+        inviteLink: getAuthURL(),
         langfuseCloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
       }),
     );

web/src/features/filters/components/multi-select.tsx

@@ -118,9 +118,9 @@ export function MultiSelect({
                     >
                       <Check className={cn("h-4 w-4")} />
                     </div>
-                    <span>{option.value}</span>
+                    <span className="overflow-x-scroll">{option.value}</span>
                     {option.count !== undefined ? (
-                      <span className="ml-auto flex h-4 w-4 items-center justify-center font-mono text-xs">
+                      <span className="ml-auto flex h-4 w-4 items-center justify-center pl-1 font-mono text-xs">
                         {option.count}
                       </span>
                     ) : null}

web/src/features/llm-api-key/types.ts

@@ -1,13 +1,8 @@
 import { z } from "zod";
 import { ModelProvider } from "@langfuse/shared";
 
-export const ZodModelProvider = z.enum([
-  ModelProvider.Anthropic,
-  ModelProvider.OpenAI,
-]);
-
 export const CreateLlmApiKey = z.object({
   projectId: z.string(),
   secretKey: z.string().min(1),
-  provider: z.literal(ModelProvider.OpenAI),
+  provider: z.nativeEnum(ModelProvider),
 });

web/src/features/manual-scoring/components/ManualScoreButton.tsx

@@ -77,13 +77,13 @@ export function ManualScoreButton({
       utils.sessions.invalidate(),
     ]);
   };
-  const mutCreateScore = api.scores.createReviewScore.useMutation({
+  const mutCreateScore = api.scores.createAnnotationScore.useMutation({
     onSuccess,
   });
-  const mutUpdateScore = api.scores.updateReviewScore.useMutation({
+  const mutUpdateScore = api.scores.updateAnnotationScore.useMutation({
     onSuccess,
   });
-  const mutDeleteScore = api.scores.deleteReviewScore.useMutation({
+  const mutDeleteScore = api.scores.deleteAnnotationScore.useMutation({
     onSuccess,
   });
 

web/src/features/models/components/NewModelForm.tsx

@@ -28,6 +28,7 @@ import { api } from "@/src/utils/api";
 import { zodResolver } from "@hookform/resolvers/zod";
 import { JsonEditor } from "@/src/components/json-editor";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import Link from "next/link";
 
 const formSchema = z.object({
   modelName: z.string().min(1),
@@ -255,11 +256,33 @@ export const NewModelForm = (props: {
             name="inputPrice"
             render={({ field }) => (
               <FormItem>
-                <FormLabel>Input price (USD)</FormLabel>
+                <FormLabel>
+                  Input price (USD per{" "}
+                  {form.getValues("unit").toLowerCase().replace(/s$/, "")})
+                </FormLabel>
                 <FormControl>
                   <Input {...field} type="number" />
                 </FormControl>
-                <FormDescription>Cost per input unit.</FormDescription>
+                {field.value !== null && field.value !== "" ? (
+                  <FormDescription>
+                    <ul className="font-mono text-xs">
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1000).toFixed(4)} USD
+                        / 1k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 100_000).toFixed(4)}{" "}
+                        USD / 100k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1_000_000).toFixed(
+                          4,
+                        )}{" "}
+                        USD / 1M {form.getValues("unit").toLowerCase()}
+                      </li>
+                    </ul>
+                  </FormDescription>
+                ) : null}
                 <FormMessage />
               </FormItem>
             )}
@@ -269,11 +292,33 @@ export const NewModelForm = (props: {
             name="outputPrice"
             render={({ field }) => (
               <FormItem>
-                <FormLabel>Output price (USD)</FormLabel>
+                <FormLabel>
+                  Output price (USD per{" "}
+                  {form.getValues("unit").toLowerCase().replace(/s$/, "")})
+                </FormLabel>
                 <FormControl>
                   <Input {...field} type="number" />
                 </FormControl>
-                <FormDescription>Cost per output unit.</FormDescription>
+                {field.value !== null && field.value !== "" ? (
+                  <FormDescription>
+                    <ul className="font-mono text-xs">
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1000).toFixed(4)} USD
+                        / 1k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 100_000).toFixed(4)}{" "}
+                        USD / 100k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1_000_000).toFixed(
+                          4,
+                        )}{" "}
+                        USD / 1M {form.getValues("unit").toLowerCase()}
+                      </li>
+                    </ul>
+                  </FormDescription>
+                ) : null}
                 <FormMessage />
               </FormItem>
             )}
@@ -283,12 +328,34 @@ export const NewModelForm = (props: {
             name="totalPrice"
             render={({ field }) => (
               <FormItem>
-                <FormLabel>Total price (USD)</FormLabel>
+                <FormLabel>
+                  Total price (USD per{" "}
+                  {form.getValues("unit").toLowerCase().replace(/s$/, "")})
+                </FormLabel>
                 <FormControl>
                   <Input {...field} type="number" />
                 </FormControl>
                 <FormDescription>
-                  Cost per unit, if no separate input/output prices.
+                  {field.value !== null && field.value !== "" ? (
+                    <ul className="mt-2 font-mono text-xs">
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1000).toFixed(4)} USD
+                        / 1k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 100_000).toFixed(4)}{" "}
+                        USD / 100k {form.getValues("unit").toLowerCase()}
+                      </li>
+                      <li>
+                        {(parseFloat(field.value ?? "0") * 1_000_000).toFixed(
+                          4,
+                        )}{" "}
+                        USD / 1M {form.getValues("unit").toLowerCase()}
+                      </li>
+                    </ul>
+                  ) : (
+                    "Enter total price only if no separate input and output prices are provided."
+                  )}
                 </FormDescription>
                 <FormMessage />
               </FormItem>
@@ -328,7 +395,15 @@ export const NewModelForm = (props: {
                 Optionally, Langfuse can tokenize the input and output of a
                 generation if no unit counts are ingested. This is useful for
                 e.g. streamed OpenAI completions. For details on the supported
-                tokenizers, see the docs.
+                tokenizers, see the{" "}
+                <Link
+                  href="https://langfuse.com/docs/model-usage-and-cost"
+                  className="underline"
+                  target="_blank"
+                >
+                  docs
+                </Link>
+                .
               </FormDescription>
               <FormMessage />
             </FormItem>
@@ -346,8 +421,15 @@ export const NewModelForm = (props: {
                   onChange={field.onChange}
                 />
                 <FormDescription>
-                  The config for the tokenizer. Required for openai. See the
-                  docs for details.
+                  The config for the tokenizer. Required for openai. See the{" "}
+                  <Link
+                    href="https://langfuse.com/docs/model-usage-and-cost"
+                    className="underline"
+                    target="_blank"
+                  >
+                    docs
+                  </Link>{" "}
+                  for details.
                 </FormDescription>
                 <FormMessage />
               </FormItem>

web/src/features/notifications/ErrorNotification.tsx

@@ -0,0 +1,79 @@
+import { Button } from "@/src/components/ui/button";
+import { AlertTriangle, X } from "lucide-react";
+import {
+  chatAvailable,
+  sendUserChatMessage,
+} from "@/src/features/support-chat/chat";
+
+interface ErrorNotificationProps {
+  error: string;
+  description: string;
+  cause?: string;
+  dismissToast: (t?: string | number | undefined) => void;
+  toast: string | number;
+  path?: string;
+}
+
+export const ErrorNotification: React.FC<ErrorNotificationProps> = ({
+  error,
+  description,
+  cause,
+  dismissToast,
+  toast,
+  path,
+}) => {
+  const handleReportIssueClick = () => {
+    if (chatAvailable) {
+      const currentUrl = window.location.href;
+      const message = `I received the following error:\n\nError: ${error}\nDescription: ${description}\n ${cause ? `Cause: ${cause}\n` : ""}${path ? `Path: ${path}\n` : ""}URL: ${currentUrl}`;
+      sendUserChatMessage(message);
+      dismissToast(toast);
+    }
+  };
+
+  return (
+    <div className="flex justify-between">
+      <div className="flex min-w-[300px] flex-1 flex-col gap-2">
+        <div className="flex items-center gap-2">
+          <AlertTriangle size={20} className="text-destructive-foreground" />
+          <div className="m-0 text-sm font-medium leading-tight text-destructive-foreground">
+            {error}
+          </div>
+        </div>
+        {description && (
+          <div className="text-sm leading-tight text-destructive-foreground">
+            {description}
+          </div>
+        )}
+        {path && (
+          <div className="text-sm leading-tight text-destructive-foreground">
+            Path: {path}
+          </div>
+        )}
+        {cause && (
+          <div className="max-h-32 overflow-y-auto text-sm leading-tight text-destructive-foreground">
+            {cause}
+          </div>
+        )}
+        {chatAvailable && (
+          <Button
+            variant="errorNotification"
+            size={"sm"}
+            onClick={() => {
+              handleReportIssueClick();
+            }}
+          >
+            Report issue to Langfuse team
+          </Button>
+        )}
+      </div>
+      <button
+        className="flex h-6 w-6 cursor-pointer items-start justify-end border-none bg-transparent p-0 text-destructive-foreground transition-colors duration-200"
+        onClick={() => dismissToast(toast)}
+        aria-label="Close"
+      >
+        <X size={14} />
+      </button>
+    </div>
+  );
+};

web/src/features/notifications/Notification.tsx

@@ -15,7 +15,7 @@ interface NotificationProps {
   toast: string | number;
 }
 
-const Notification: React.FC<NotificationProps> = ({
+export const Notification: React.FC<NotificationProps> = ({
   notification,
   setLastSeenId,
   dismissToast,
@@ -50,5 +50,3 @@ const Notification: React.FC<NotificationProps> = ({
     </div>
   );
 };
-
-export default Notification;

web/src/features/notifications/checkNotifications.tsx

@@ -2,7 +2,8 @@ import { useEffect } from "react";
 import Link from "next/link";
 import { toast } from "sonner";
 import useLocalStorage from "@/src/components/useLocalStorage";
-import Notification, {
+import {
+  Notification,
   type TNotification,
 } from "@/src/features/notifications/Notification";
 import { Button } from "@/src/components/ui/button";

web/src/features/notifications/showErrorToast.tsx

@@ -0,0 +1,31 @@
+import { toast } from "sonner";
+import { ErrorNotification } from "@/src/features/notifications/ErrorNotification";
+
+export const showErrorToast = (
+  error: string,
+  description: string,
+  cause?: string,
+  path?: string,
+) => {
+  toast.custom(
+    (t) => (
+      <ErrorNotification
+        error={error}
+        description={description}
+        cause={cause}
+        path={path}
+        dismissToast={toast.dismiss}
+        toast={t}
+      />
+    ),
+    {
+      duration: Infinity,
+      style: {
+        padding: "1rem",
+        border: "1px solid hsl(var(--destructive))",
+        borderRadius: "0.5rem",
+        backgroundColor: "hsl(var(--destructive))",
+      },
+    },
+  );
+};

web/src/features/posthog-analytics/usePostHogClientCapture.ts

@@ -91,6 +91,7 @@ const events = {
     "create_new_button_click",
   ],
   onboarding: ["code_example_tab_switch"],
+  user_settings: ["theme_changed"],
   project_settings: [
     "project_delete",
     "rename_form_submit",

web/src/features/prompts/components/prompt-detail.tsx

@@ -169,6 +169,7 @@ export const PromptDetail = () => {
                 promptName={prompt.name}
                 tags={prompt.tags}
                 availableTags={allTags}
+                className="flex-wrap"
               />
             </div>
           </div>

web/src/features/prompts/server/actions/createPrompt.ts

@@ -6,6 +6,8 @@ import { ValidationError } from "@langfuse/shared";
 import { jsonSchema } from "@/src/utils/zod";
 import { type PrismaClient } from "@langfuse/shared/src/db";
 import { LATEST_PROMPT_LABEL } from "@/src/features/prompts/constants";
+import { removeLabelsFromPreviousPromptVersions } from "@/src/features/prompts/server/utils/updatePromptLabels";
+import { updatePromptTagsOnAllVersions } from "@/src/features/prompts/server/utils/updatePromptTags";
 
 export type CreatePromptParams = CreatePromptTRPCType & {
   createdBy: string;
@@ -21,6 +23,7 @@ export const createPrompt = async ({
   config,
   createdBy,
   prisma,
+  tags,
 }: CreatePromptParams) => {
   const latestPrompt = await prisma.prompt.findFirst({
     where: { projectId, name },
@@ -35,14 +38,8 @@ export const createPrompt = async ({
 
   const finalLabels = [...labels, LATEST_PROMPT_LABEL]; // Newly created prompts are always labeled as 'latest'
 
-  const previousLabeledPrompts = await prisma.prompt.findMany({
-    where: {
-      projectId,
-      name,
-      labels: { hasSome: finalLabels },
-    },
-    orderBy: [{ version: "desc" }],
-  });
+  // If tags are undefined, use the tags from the latest prompt version
+  const finalTags = [...new Set(tags ?? latestPrompt?.tags ?? [])];
 
   const create = [
     prisma.prompt.create({
@@ -52,7 +49,7 @@ export const createPrompt = async ({
         createdBy,
         labels: [...new Set(finalLabels)], // Ensure labels are unique
         type,
-        tags: latestPrompt?.tags,
+        tags: finalTags,
         version: latestPrompt?.version ? latestPrompt.version + 1 : 1,
         project: { connect: { id: projectId } },
         config: jsonSchema.parse(config),
@@ -62,18 +59,28 @@ export const createPrompt = async ({
 
   if (finalLabels.length > 0)
     // If we're creating a new labeled prompt, we must remove those labels on previous prompts since labels are unique
-    previousLabeledPrompts.forEach((prevPrompt) => {
-      create.push(
-        prisma.prompt.update({
-          where: { id: prevPrompt.id },
-          data: {
-            labels: prevPrompt.labels.filter(
-              (prevLabel) => !finalLabels.includes(prevLabel),
-            ),
-          },
-        }),
-      );
-    });
+    create.push(
+      ...(await removeLabelsFromPreviousPromptVersions({
+        prisma,
+        projectId,
+        promptName: name,
+        labelsToRemove: finalLabels,
+      })),
+    );
+
+  const haveTagsChanged =
+    JSON.stringify([...new Set(finalTags)].sort()) !==
+    JSON.stringify([...new Set(latestPrompt?.tags)].sort());
+  if (haveTagsChanged)
+    // If we're creating a new prompt with tags, we must update those tags on previous prompts since tags are consistent across versions
+    create.push(
+      ...(await updatePromptTagsOnAllVersions({
+        prisma,
+        projectId,
+        promptName: name,
+        tags: finalTags,
+      })),
+    );
 
   const [createdPrompt] = await prisma.$transaction(create);
 

web/src/features/prompts/server/utils/updatePromptLabels.ts

@@ -0,0 +1,33 @@
+import { type PrismaClient } from "@langfuse/shared/src/db";
+
+export const removeLabelsFromPreviousPromptVersions = async ({
+  prisma,
+  projectId,
+  promptName,
+  labelsToRemove,
+}: {
+  prisma: PrismaClient;
+  projectId: string;
+  promptName: string;
+  labelsToRemove: string[];
+}) => {
+  const previouslyLabeledPrompts = await prisma.prompt.findMany({
+    where: {
+      projectId,
+      name: promptName,
+      labels: { hasSome: labelsToRemove },
+    },
+    orderBy: [{ version: "desc" }],
+  });
+
+  return previouslyLabeledPrompts.map((prevPrompt) =>
+    prisma.prompt.update({
+      where: { id: prevPrompt.id },
+      data: {
+        labels: prevPrompt.labels.filter(
+          (prevLabel) => !labelsToRemove.includes(prevLabel),
+        ),
+      },
+    }),
+  );
+};

web/src/features/prompts/server/utils/updatePromptTags.ts

@@ -0,0 +1,28 @@
+import { type PrismaClient } from "@langfuse/shared/src/db";
+
+export const updatePromptTagsOnAllVersions = async ({
+  prisma,
+  projectId,
+  promptName,
+  tags,
+}: {
+  prisma: PrismaClient;
+  projectId: string;
+  promptName: string;
+  tags: string[];
+}) => {
+  const previousVersions = await prisma.prompt.findMany({
+    where: { projectId, name: promptName },
+  });
+
+  if (previousVersions.length === 0) return [];
+
+  return previousVersions.map((prevVersion) =>
+    prisma.prompt.update({
+      where: { id: prevVersion.id },
+      data: {
+        tags: [...new Set(tags)], // Ensure tags are unique
+      },
+    }),
+  );
+};

web/src/features/prompts/server/utils/validation.ts

@@ -27,6 +27,7 @@ export const CreateTextPromptSchema = z.object({
   type: z.literal(PromptType.Text).optional(),
   prompt: z.string(),
   config: jsonSchema.nullable().default({}),
+  tags: z.array(z.string()).nullish(),
 });
 
 export const CreateChatPromptSchema = z.object({
@@ -35,6 +36,7 @@ export const CreateChatPromptSchema = z.object({
   type: z.literal(PromptType.Chat),
   prompt: z.array(ChatMessageSchema),
   config: jsonSchema.nullable().default({}),
+  tags: z.array(z.string()).nullish(),
 });
 
 export const CreatePromptSchema = z.union([

web/src/features/public-api/components/LLMApiKeyList.tsx

@@ -59,7 +59,7 @@ export function LlmApiKeyList(props: { projectId: string }) {
       projectId: props.projectId,
     },
     {
-      enabled: hasAccess,
+      enabled: hasAccess && env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined,
     },
   );
 
@@ -175,7 +175,7 @@ function DeleteApiKeyButton(props: { projectId: string; apiKeyId: string }) {
 
 const formSchema = z.object({
   secretKey: z.string().min(1),
-  provider: z.literal(ModelProvider.OpenAI),
+  provider: z.nativeEnum(ModelProvider),
 });
 
 export function CreateLlmApiKeyComponent(props: {
@@ -277,7 +277,7 @@ export function CreateLlmApiKeyComponent(props: {
                     <Select
                       defaultValue={field.value}
                       onValueChange={(value) =>
-                        field.onChange(value as ModelProvider[number])
+                        field.onChange(value as ModelProvider)
                       }
                     >
                       <FormControl>
@@ -286,13 +286,7 @@ export function CreateLlmApiKeyComponent(props: {
                         </SelectTrigger>
                       </FormControl>
                       <SelectContent>
-                        {Array.from(
-                          new Set(
-                            evalLLMModels.map(
-                              (models) => models.provider.value,
-                            ),
-                          ),
-                        ).map((provider) => (
+                        {Object.values(ModelProvider).map((provider) => (
                           <SelectItem value={provider} key={provider}>
                             {provider}
                           </SelectItem>

web/src/features/tag/components/TagMananger.tsx

@@ -18,6 +18,7 @@ type TagManagerProps = {
   hasAccess: boolean;
   isLoading: boolean;
   mutateTags: (value: string[]) => void;
+  className?: string;
 };
 
 const TagManager = ({
@@ -26,6 +27,7 @@ const TagManager = ({
   hasAccess,
   isLoading,
   mutateTags,
+  className,
 }: TagManagerProps) => {
   const {
     selectedTags,
@@ -59,7 +61,7 @@ const TagManager = ({
   return (
     <Popover onOpenChange={(open) => handlePopoverChange(open)}>
       <PopoverTrigger className="select-none" asChild>
-        <div className="flex gap-x-2 gap-y-1">
+        <div className={cn("flex gap-x-2 gap-y-1", className)}>
           <TagList selectedTags={selectedTags} isLoading={isLoading} />
         </div>
       </PopoverTrigger>

web/src/features/tag/components/TagPromptDetailsPopover.tsx

@@ -3,12 +3,14 @@ import { api } from "@/src/utils/api";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type RouterOutput } from "@/src/utils/types";
 import TagManager from "@/src/features/tag/components/TagMananger";
+import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
 
 type TagPromptDetailsPopoverProps = {
   tags: string[];
   availableTags: string[];
   projectId: string;
   promptName: string;
+  className?: string;
 };
 
 export function TagPromptDetailsPopover({
@@ -16,6 +18,7 @@ export function TagPromptDetailsPopover({
   availableTags,
   projectId,
   promptName,
+  className,
 }: TagPromptDetailsPopoverProps) {
   const [isLoading, setIsLoading] = useState(false);
   const hasAccess = useHasAccess({ projectId, scope: "objects:tag" });
@@ -34,6 +37,7 @@ export function TagPromptDetailsPopover({
     },
     onError: (err, _newTags, context) => {
       setIsLoading(false);
+      trpcErrorToast(err);
       // Rollback to the previous value if mutation fails
       utils.prompts.allVersions.setData(
         { projectId: projectId, name: promptName },
@@ -78,6 +82,7 @@ export function TagPromptDetailsPopover({
       hasAccess={hasAccess}
       isLoading={isLoading}
       mutateTags={mutateTags}
+      className={className}
     />
   );
 }

web/src/features/tag/components/TagPromptPopover.tsx

@@ -3,6 +3,7 @@ import { api } from "@/src/utils/api";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type RouterOutput, type RouterInput } from "@/src/utils/types";
 import TagManager from "@/src/features/tag/components/TagMananger";
+import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
 
 type TagPromptPopverProps = {
   tags: string[];
@@ -10,6 +11,7 @@ type TagPromptPopverProps = {
   projectId: string;
   promptName: string;
   promptsFilter: RouterInput["prompts"]["all"];
+  className?: string;
 };
 
 export function TagPromptPopover({
@@ -18,6 +20,7 @@ export function TagPromptPopover({
   projectId,
   promptName,
   promptsFilter,
+  className,
 }: TagPromptPopverProps) {
   const [isLoading, setIsLoading] = useState(false);
   const hasAccess = useHasAccess({ projectId, scope: "objects:tag" });
@@ -32,7 +35,7 @@ export function TagPromptPopover({
     },
     onError: (err, _newTags, context) => {
       utils.prompts.all.setData(promptsFilter, context?.prevPrompt);
-      console.error(err);
+      trpcErrorToast(err);
       setIsLoading(false);
     },
     onSettled: (data, error, { name, tags }) => {
@@ -66,6 +69,7 @@ export function TagPromptPopover({
       hasAccess={hasAccess}
       isLoading={isLoading}
       mutateTags={mutateTags}
+      className={className}
     />
   );
 }

web/src/features/tag/components/TagTraceDetailsPopover.tsx

@@ -3,12 +3,14 @@ import { api } from "@/src/utils/api";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type RouterOutput } from "@/src/utils/types";
 import TagManager from "@/src/features/tag/components/TagMananger";
+import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
 
 type TagTraceDetailsPopoverProps = {
   tags: string[];
   availableTags: string[];
   projectId: string;
   traceId: string;
+  className?: string;
 };
 
 export function TagTraceDetailsPopover({
@@ -16,6 +18,7 @@ export function TagTraceDetailsPopover({
   availableTags,
   projectId,
   traceId,
+  className,
 }: TagTraceDetailsPopoverProps) {
   const [isLoading, setIsLoading] = useState(false);
   const hasAccess = useHasAccess({ projectId, scope: "objects:tag" });
@@ -32,6 +35,7 @@ export function TagTraceDetailsPopover({
     },
     onError: (err, _newTags, context) => {
       setIsLoading(false);
+      trpcErrorToast(err);
       // Rollback to the previous value if mutation fails
       utils.traces.byId.setData({ traceId }, context?.prev);
     },
@@ -69,6 +73,7 @@ export function TagTraceDetailsPopover({
       hasAccess={hasAccess}
       isLoading={isLoading}
       mutateTags={mutateTags}
+      className={className}
     />
   );
 }

web/src/features/tag/components/TagTracePopver.tsx

@@ -3,6 +3,7 @@ import { api } from "@/src/utils/api";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type RouterOutput, type RouterInput } from "@/src/utils/types";
 import TagManager from "@/src/features/tag/components/TagMananger";
+import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
 
 type TagTracePopoverProps = {
   tags: string[];
@@ -10,6 +11,7 @@ type TagTracePopoverProps = {
   projectId: string;
   traceId: string;
   tracesFilter: RouterInput["traces"]["all"];
+  className?: string;
 };
 
 export function TagTracePopover({
@@ -18,6 +20,7 @@ export function TagTracePopover({
   projectId,
   traceId,
   tracesFilter,
+  className,
 }: TagTracePopoverProps) {
   const [isLoading, setIsLoading] = useState(false);
   const hasAccess = useHasAccess({ projectId, scope: "objects:tag" });
@@ -32,7 +35,7 @@ export function TagTracePopover({
     },
     onError: (err, _newTags, context) => {
       utils.traces.all.setData(tracesFilter, context?.prevTrace);
-      console.log("error", err);
+      trpcErrorToast(err);
       setIsLoading(false);
     },
     onSettled: (data, error, { traceId, tags }) => {
@@ -68,6 +71,7 @@ export function TagTracePopover({
       hasAccess={hasAccess}
       isLoading={isLoading}
       mutateTags={mutateTags}
+      className={className}
     />
   );
 }

web/src/features/theming/ThemeToggle.tsx

@@ -2,10 +2,11 @@ import * as React from "react";
 import { Monitor, Moon, Sun } from "lucide-react";
 import { useTheme } from "next-themes";
 import { cn } from "@/src/utils/tailwind";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export function ThemeToggle() {
   const { theme, setTheme } = useTheme();
-
+  const capture = usePostHogClientCapture();
   return (
     <div className="ml-auto flex items-center space-x-1">
       <div title="Light mode">
@@ -17,6 +18,9 @@ export function ThemeToggle() {
           onClick={(e) => {
             e.preventDefault();
             setTheme("light");
+            capture("user_settings:theme_changed", {
+              theme: "light",
+            });
           }}
         />
       </div>
@@ -29,6 +33,9 @@ export function ThemeToggle() {
           onClick={(e) => {
             e.preventDefault();
             setTheme("dark");
+            capture("user_settings:theme_changed", {
+              theme: "dark",
+            });
           }}
         />
       </div>
@@ -41,6 +48,9 @@ export function ThemeToggle() {
           onClick={(e) => {
             e.preventDefault();
             setTheme("system");
+            capture("user_settings:theme_changed", {
+              theme: "system",
+            });
           }}
         />
       </div>

web/src/features/usage-metering/ProjectUsageChart.tsx

@@ -34,7 +34,7 @@ export const ProjectUsageChart: React.FC<{ projectId: string }> = ({
 
   return (
     <div>
-      <Header title="Usage" level="h3" />
+      <Header title="Usage & Billing" level="h3" />
       <Card className="p-4 lg:w-1/2">
         {usage.data !== undefined && (
           <>
@@ -81,7 +81,7 @@ export const ProjectUsageChart: React.FC<{ projectId: string }> = ({
                   level="h3"
                   actionButtons={
                     <Button variant="secondary" asChild>
-                      <Link href="https://langfuse.com/pricing">
+                      <Link href="https://langfuse.com/pricing" target="_blank">
                         Pricing page ↗
                       </Link>
                     </Button>
@@ -98,6 +98,11 @@ export const ProjectUsageChart: React.FC<{ projectId: string }> = ({
             </Link>
           </Button>
         )}
+        <Button variant="secondary" asChild>
+          <Link href="https://langfuse.com/pricing" target="_blank">
+            Pricing page ↗
+          </Link>
+        </Button>
         <div className="inline-block text-sm text-muted-foreground">
           Current plan: {plan}
         </div>