chore: release v2.46.0

Fixes #2107

.env.prod.example

@@ -58,6 +58,11 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_AUTH0_CLIENT_SECRET=
 # AUTH_AUTH0_ISSUER=
 # AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false
+# AUTH_COGNITO_CLIENT_ID=
+# AUTH_COGNITO_CLIENT_SECRET=
+# AUTH_COGNITO_ISSUER=
+# AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+
 
 # Transactional email, optional
 # Defines the email address to use as the from address.

.github/workflows/porter_stack_worker.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - main
+name: Deploy to worker
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "3959"
+        PORTER_DEPLOYMENT_TARGET_ID: d2bb23e4-1d77-48f2-a113-383d076959b3
+        PORTER_HOST: https://dashboard.getporter.dev
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: worker
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_3959 }}

careers

@@ -3,9 +3,9 @@ Join us in scaling Langfuse in Berlin, Germany. We are an open source company, w
 
 _Open Roles_
 
-- Backend Engineer, 70-110k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
-- Product Engineer, 70-110k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
-- Developer Advocate, 60-100k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+- Backend Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
+- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
 
 _More Info_
 

ee/src/sso/index.ts

@@ -2,6 +2,7 @@ import { type Provider } from "next-auth/providers/index";
 import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
 import OktaProvider from "next-auth/providers/okta";
+import CognitoProvider from "next-auth/providers/cognito";
 import Auth0Provider from "next-auth/providers/auth0";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { isEeAvailable } from "..";
@@ -144,6 +145,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "cognito")
+    return CognitoProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line no-unused-vars

ee/src/sso/types.ts

@@ -64,11 +64,24 @@ export const AzureAdProviderSchema = base.extend({
     .nullish(),
 });
 
+export const CognitoProviderSchema = base.extend({
+  authProvider: z.literal("cognito"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
@@ -76,6 +89,7 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   Auth0ProviderSchema,
   OktaProviderSchema,
   AzureAdProviderSchema,
+  CognitoProviderSchema,
 ]);
 
 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

fern/apis/server/definition/commons.yml

@@ -209,8 +209,8 @@ types:
       - ARCHIVED
   ScoreSource:
     enum:
+      - ANNOTATION
       - API
-      - REVIEW
       - EVAL
 
 errors:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.45.0",
+  "version": "2.46.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,

packages/shared/prisma/generated/types.ts

@@ -25,11 +25,16 @@ export const ObservationLevel = {
 } as const;
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
+    ANNOTATION: "ANNOTATION",
     API: "API",
-    REVIEW: "REVIEW",
     EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
+export const ScoreDataType = {
+    CATEGORICAL: "CATEGORICAL",
+    NUMERIC: "NUMERIC"
+} as const;
+export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
 export const PricingUnit = {
     PER_1000_TOKENS: "PER_1000_TOKENS",
     PER_1000_CHARS: "PER_1000_CHARS"
@@ -351,6 +356,22 @@ export type Score = {
     comment: string | null;
     trace_id: string;
     observation_id: string | null;
+    config_id: string | null;
+    string_value: string | null;
+    data_type: Generated<ScoreDataType>;
+};
+export type ScoreConfig = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    data_type: ScoreDataType;
+    is_archived: Generated<boolean>;
+    min_value: number | null;
+    max_value: number | null;
+    categories: unknown | null;
+    description: string | null;
 };
 export type Session = {
     id: string;
@@ -450,6 +471,7 @@ export type DB = {
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
+    score_configs: ScoreConfig;
     scores: Score;
     Session: Session;
     sso_configs: SsoConfig;

packages/shared/prisma/migrations/20240523142425_score_config_add_table/migration.sql

@@ -0,0 +1,42 @@
+-- CreateEnum
+CREATE TYPE "ScoreDataType" AS ENUM ('CATEGORICAL', 'NUMERIC');
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "config_id" TEXT,
+ADD COLUMN     "data_type" "ScoreDataType" NOT NULL DEFAULT 'NUMERIC',
+ADD COLUMN     "string_value" TEXT;
+
+-- CreateTable
+CREATE TABLE "score_configs" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "data_type" "ScoreDataType" NOT NULL,
+    "is_archived" BOOLEAN NOT NULL DEFAULT false,
+    "min_value" DOUBLE PRECISION,
+    "max_value" DOUBLE PRECISION,
+    "categories" JSONB,
+    "description" TEXT,
+
+    CONSTRAINT "score_configs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "score_configs_data_type_idx" ON "score_configs"("data_type");
+
+-- CreateIndex
+CREATE INDEX "score_configs_is_archived_idx" ON "score_configs"("is_archived");
+
+-- CreateIndex
+CREATE INDEX "score_configs_project_id_idx" ON "score_configs"("project_id");
+
+-- CreateIndex
+CREATE INDEX "score_configs_categories_idx" ON "score_configs"("categories");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "score_configs_id_project_id_key" ON "score_configs"("id", "project_id");
+
+-- AddForeignKey
+ALTER TABLE "score_configs" ADD CONSTRAINT "score_configs_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240523142524_scores_add_config_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_config_id_idx" ON "scores"("config_id");

packages/shared/prisma/migrations/20240523142610_scores_add_fk_scores_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_config_id_fkey" FOREIGN KEY ("config_id") REFERENCES "score_configs"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240524154058_scores_source_enum_add_annotation/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'ANNOTATION';

packages/shared/prisma/migrations/20240524156058_scores_source_backfill_annotation_for_review/migration.sql

@@ -0,0 +1,4 @@
+-- Backfill the scores source for 'REVIEW' to be 'ANNOTATION'
+UPDATE "scores"
+SET "source" = 'ANNOTATION'::"ScoreSource"
+WHERE "source" = 'REVIEW'::"ScoreSource";

packages/shared/prisma/migrations/20240524165931_scores_source_enum_drop_review/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - The values [REVIEW] on the enum `ScoreSource` will be removed. If these variants are still used in the database, this will fail.
+
+*/
+-- AlterEnum
+BEGIN;
+CREATE TYPE "ScoreSource_new" AS ENUM ('ANNOTATION', 'API', 'EVAL');
+ALTER TABLE "scores" ALTER COLUMN "source" TYPE "ScoreSource_new" USING ("source"::text::"ScoreSource_new");
+ALTER TYPE "ScoreSource" RENAME TO "ScoreSource_old";
+ALTER TYPE "ScoreSource_new" RENAME TO "ScoreSource";
+DROP TYPE "ScoreSource_old";
+COMMIT;

packages/shared/prisma/migrations/20240524190433_job_executions_add_fk_index_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_configuration_id_idx" ON "job_executions"("job_configuration_id");

packages/shared/prisma/migrations/20240524190434_job_executions_add_fk_index_score_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_output_score_id_idx" ON "job_executions"("job_output_score_id");

packages/shared/prisma/migrations/20240524190435_job_executions_add_fk_index_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_input_trace_id_idx" ON "job_executions"("job_input_trace_id");

packages/shared/prisma/migrations/20240524190436_job_executions_index_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_created_at_idx" ON "job_executions"("created_at");

packages/shared/prisma/schema.prisma

@@ -114,6 +114,7 @@ model Project {
     LlmApiKeys         LlmApiKeys[]
     PosthogIntegration PosthogIntegration[]
     Score              Score[]
+    scoreConfig        ScoreConfig[]
 
     @@map("projects")
 }
@@ -403,13 +404,18 @@ model Score {
     comment       String?
     traceId       String         @map("trace_id")
     observationId String?        @map("observation_id")
+    configId      String?        @map("config_id")
+    stringValue   String?        @map("string_value")
+    dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
     JobExecution  JobExecution[]
+    scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
 
     @@unique([id, projectId]) // used for upserts via prisma
     @@index(timestamp)
     @@index([value])
     @@index([projectId])
     @@index([authorUserId])
+    @@index([configId])
     @@index([traceId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([source])
@@ -417,11 +423,39 @@ model Score {
 }
 
 enum ScoreSource {
+    ANNOTATION
     API
-    REVIEW
     EVAL
 }
 
+model ScoreConfig {
+    id          String        @id @default(cuid())
+    createdAt   DateTime      @default(now()) @map("created_at")
+    updatedAt   DateTime      @default(now()) @updatedAt @map("updated_at")
+    projectId   String        @map("project_id")
+    project     Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    name        String
+    dataType    ScoreDataType @map("data_type")
+    isArchived  Boolean       @default(false) @map("is_archived")
+    minValue    Float?        @map("min_value")
+    maxValue    Float?        @map("max_value")
+    categories  Json?         @map("categories")
+    description String?
+    score       Score[]
+
+    @@unique([id, projectId]) // used for upserts via prisma
+    @@index([dataType])
+    @@index([isArchived])
+    @@index([projectId])
+    @@index([categories])
+    @@map("score_configs")
+}
+
+enum ScoreDataType {
+    CATEGORICAL
+    NUMERIC
+}
+
 enum PricingUnit {
     PER_1000_TOKENS
     PER_1000_CHARS
@@ -712,6 +746,10 @@ model JobExecution {
     @@index([projectId, status])
     @@index([projectId, id])
     @@index([projectId])
+    @@index([jobConfigurationId])
+    @@index([jobOutputScoreId])
+    @@index([jobInputTraceId])
+    @@index([createdAt])
     @@map("job_executions")
 }
 

packages/shared/prisma/seed.ts

@@ -12,6 +12,7 @@ import { chunk } from "lodash";
 import { v4 } from "uuid";
 import { ModelUsageUnit } from "../src";
 import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
+import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -165,6 +166,24 @@ async function main() {
 
     await uploadObjects(traces, observations, scores, sessions, events);
 
+    // If openai key is in environment, add it to the projects LLM API keys
+    const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+
+    if (OPENAI_API_KEY) {
+      await prisma.llmApiKeys.create({
+        data: {
+          projectId: project1.id,
+          secretKey: encrypt(OPENAI_API_KEY),
+          displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
+          provider: "openai",
+        },
+      });
+    } else {
+      console.warn(
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key."
+      );
+    }
+
     // add eval objects
     const evalTemplate = await prisma.evalTemplate.upsert({
       where: {
@@ -500,7 +519,7 @@ function createObjects(
               name: "manual-score",
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
-              source: ScoreSource.REVIEW,
+              source: ScoreSource.ANNOTATION,
               projectId,
               authorUserId: `user-${i}`,
             },

pnpm-lock.yaml

@@ -466,8 +466,8 @@ importers:
         specifier: ^7.51.5
         version: 7.51.5(react@18.2.0)
       react-icons:
-        specifier: ^5.0.1
-        version: 5.0.1(react@18.2.0)
+        specifier: ^5.2.1
+        version: 5.2.1(react@18.2.0)
       react-responsive:
         specifier: ^10.0.0
         version: 10.0.0(react@18.2.0)
@@ -15018,8 +15018,8 @@ packages:
       react: 18.2.0
     dev: false
 
-  /react-icons@5.0.1(react@18.2.0):
-    resolution: {integrity: sha512-WqLZJ4bLzlhmsvme6iFdgO8gfZP17rfjYEJ2m9RsZjZ+cc4k1hTzknEz63YS1MeT50kVzoa1Nz36f4BEx+Wigw==}
+  /react-icons@5.2.1(react@18.2.0):
+    resolution: {integrity: sha512-zdbW5GstTzXaVKvGSyTaBalt7HSfuK5ovrzlpyiWHAFXndXTdd/1hdDHI4xBM1Mn7YriT6aqESucFl9kEXzrdw==}
     peerDependencies:
       react: '*'
     dependencies:

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.45.0",
+  "version": "2.46.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -106,7 +106,7 @@
     "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
     "react-hook-form": "^7.51.5",
-    "react-icons": "^5.0.1",
+    "react-icons": "^5.2.1",
     "react-responsive": "^10.0.0",
     "react18-json-view": "^0.2.8-canary.6",
     "sonner": "^1.4.41",

web/public/generated/api/openapi.yml

@@ -1825,8 +1825,8 @@ components:
       title: ScoreSource
       type: string
       enum:
+        - ANNOTATION
         - API
-        - REVIEW
         - EVAL
     CreateDatasetItemRequest:
       title: CreateDatasetItemRequest

web/src/__tests__/scores.servertest.ts

@@ -489,16 +489,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
         {
           id: scoreId_2,
           name: scoreName,
           value: 50.5,
         },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
       ]);
     });
     it("test operator >=", async () => {
@@ -514,16 +514,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
         {
           id: scoreId_3,
           name: scoreName,
           value: 100.8,
         },
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
       ]);
     });
     it("test operator !=", async () => {
@@ -539,16 +539,16 @@ describe("/api/public/scores API Endpoint", () => {
         totalPages: 1,
       });
       expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
         {
           id: scoreId_3,
           name: scoreName,
           value: 100.8,
         },
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
       ]);
     });
     it("test operator =", async () => {

web/src/app/api/chatCompletion/route.ts

@@ -0,0 +1,6 @@
+import chatCompletionHandler from "@/src/ee/features/playground/server/chatCompletionHandler";
+
+export const dynamic = "force-dynamic";
+export const maxDuration = 60;
+
+export const POST = chatCompletionHandler;

web/src/components/ModelParameters/LLMApiKeyComponent.tsx

@@ -0,0 +1,77 @@
+import Link from "next/link";
+
+import { Label } from "@/src/components/ui/label";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { api } from "@/src/utils/api";
+import { supportedModels, type UIModelParams } from "@langfuse/shared";
+import { ArrowTopRightIcon } from "@radix-ui/react-icons";
+
+export const LLMApiKeyComponent = (p: {
+  projectId: string;
+  modelParams: UIModelParams;
+}) => {
+  const hasAccess = useHasAccess({
+    projectId: p.projectId,
+    scope: "llmApiKeys:read",
+  });
+
+  if (!hasAccess) {
+    return (
+      <div>
+        <Label className="text-xs font-semibold">API key</Label>
+        <p className="text-sm text-muted-foreground">
+          LLM API Key only visible to Owner and Admin roles.
+        </p>
+      </div>
+    );
+  }
+
+  const apiKeys = api.llmApiKey.all.useQuery({
+    projectId: p.projectId,
+  });
+
+  if (apiKeys.isLoading) {
+    return (
+      <div>
+        <Label className="text-xs font-semibold">API key</Label>
+        <p className="text-sm text-muted-foreground">Loading...</p>
+      </div>
+    );
+  }
+
+  const model = p.modelParams.model.value;
+  const modelProvider = Object.entries(supportedModels).find((providerData) =>
+    (providerData[1] as any as string[]).includes(model),
+  )?.[0];
+
+  const apiKey = apiKeys.data?.data.find((k) => k.provider === modelProvider);
+
+  return (
+    <div className="space-y-2 text-xs">
+      <Label className="text-xs font-semibold">API key</Label>
+      <div>
+        {apiKey ? (
+          <span className="mr-2 rounded-sm bg-input p-1 text-xs">
+            {apiKey.displaySecretKey}
+          </span>
+        ) : undefined}
+      </div>
+      {/* Custom form message to include a link to the already existing prompt */}
+      {!apiKey ? (
+        <div className="flex flex-col font-medium text-destructive">
+          {`No LLM API key found for provider ${modelProvider}.`}
+
+          <Link
+            href={`/project/${p.projectId}/settings`}
+            className="flex flex-row"
+          >
+            Create a new LLM API key here. <ArrowTopRightIcon />
+          </Link>
+        </div>
+      ) : undefined}
+      <p className="text-muted-foreground">
+        The LLM API key is used for each execution and will incur costs.
+      </p>
+    </div>
+  );
+};

web/src/components/ModelParameters/index.tsx

@@ -8,6 +8,7 @@ import {
 } from "@/src/components/ui/select";
 import { Slider } from "@/src/components/ui/slider";
 import { Switch } from "@/src/components/ui/switch";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { cn } from "@/src/utils/tailwind";
 import {
   ModelProvider,
@@ -15,6 +16,8 @@ import {
   type UIModelParams,
 } from "@langfuse/shared";
 
+import { LLMApiKeyComponent } from "./LLMApiKeyComponent";
+
 export type ModelParamsContext = {
   modelParams: UIModelParams;
   availableModels?: UIModelParams[];
@@ -33,10 +36,14 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
   setModelParamEnabled,
   formDisabled = false,
 }) => {
+  const projectId = useProjectIdFromURL();
+
+  if (!projectId) return null;
+
   return (
     <div className="flex flex-col space-y-4">
       <p className="font-semibold">Model</p>
-      <div className="space-y-6">
+      <div className="space-y-4">
         <ModelParamsSelect
           title="Provider"
           modelParamsKey="provider"
@@ -104,6 +111,7 @@ export const ModelParameters: React.FC<ModelParamsContext> = ({
           tooltip="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both."
           updateModelParam={updateModelParamValue}
         />
+        <LLMApiKeyComponent {...{ projectId, modelParams }} />
       </div>
     </div>
   );

web/src/components/layouts/doc-popup.tsx

@@ -4,6 +4,7 @@ import {
   HoverCardTrigger,
 } from "@/src/components/ui/hover-card";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { cn } from "@/src/utils/tailwind";
 import { HelpCircle, Info } from "lucide-react";
 import Link from "next/link";
 
@@ -40,7 +41,10 @@ export default function DocPopup({
         }
       }}
     >
-      <HoverCardTrigger className="mx-1 cursor-pointer" asChild>
+      <HoverCardTrigger
+        className={cn("mx-1", href ? "cursor-pointer" : "cursor-default")}
+        asChild
+      >
         {href ? (
           <Link
             href={href}

web/src/components/table/use-cases/traces.tsx

@@ -547,6 +547,7 @@ export default function TracesTable({
       header: "Version",
       enableHiding: true,
       enableSorting: true,
+      defaultHidden: true,
     },
     {
       accessorKey: "release",
@@ -554,6 +555,7 @@ export default function TracesTable({
       header: "Release",
       enableHiding: true,
       enableSorting: true,
+      defaultHidden: true,
     },
     {
       accessorKey: "tags",

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.45.0";
+export const VERSION = "v2.46.0";

web/src/ee/features/evals/components/template-form.tsx

@@ -33,8 +33,6 @@ import {
   type ModelParams,
 } from "@langfuse/shared";
 import { PromptDescription } from "@/src/features/prompts/components/prompt-description";
-import Link from "next/dist/client/link";
-import { ArrowTopRightIcon } from "@radix-ui/react-icons";
 import {
   Select,
   SelectContent,
@@ -43,8 +41,6 @@ import {
   SelectValue,
 } from "@/src/components/ui/select";
 import { TEMPLATES } from "@/src/ee/features/evals/components/templates";
-import { Label } from "@/src/components/ui/label";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
 
@@ -439,10 +435,6 @@ export const InnerEvalTemplateForm = (props: {
               availableModels={[...evalLLMModels]}
               formDisabled={!props.isEditing}
             />
-            <LLMApiKeyComponent
-              projectId={props.projectId}
-              modelParams={modelParams}
-            />
           </div>
         </div>
 
@@ -465,78 +457,6 @@ export const InnerEvalTemplateForm = (props: {
   );
 };
 
-export const LLMApiKeyComponent = (p: {
-  projectId: string;
-  modelParams: UIModelParams;
-}) => {
-  const hasAccess = useHasAccess({
-    projectId: p.projectId,
-    scope: "llmApiKeys:read",
-  });
-
-  if (!hasAccess) {
-    return (
-      <div>
-        <Label>API key</Label>
-        <p className="text-sm text-muted-foreground">
-          LLM API Key only visible to Owner and Admin roles.
-        </p>
-      </div>
-    );
-  }
-
-  const apiKeys = api.llmApiKey.all.useQuery({
-    projectId: p.projectId,
-  });
-
-  if (apiKeys.isLoading) {
-    return (
-      <div>
-        <Label>API key</Label>
-        <p className="text-sm text-muted-foreground">Loading...</p>
-      </div>
-    );
-  }
-
-  const getModelProvider = (model: string) => {
-    return evalLLMModels.find((m) => m.model.value === model)?.provider.value;
-  };
-
-  const getApiKeyForModel = (model: string) => {
-    const modelProvider = getModelProvider(model);
-    return apiKeys.data?.data.find((k) => k.provider === modelProvider);
-  };
-
-  return (
-    <div>
-      <Label>API key</Label>
-      <div>
-        {getApiKeyForModel(p.modelParams.model.value) ? (
-          <span className="mr-2 rounded-sm bg-input p-1 text-xs">
-            {getApiKeyForModel(p.modelParams.model.value)?.displaySecretKey}
-          </span>
-        ) : undefined}
-      </div>
-      {/* Custom form message to include a link to the already existing prompt */}
-      {!getApiKeyForModel(p.modelParams.model.value) ? (
-        <div className="flex flex-col text-sm font-medium text-destructive">
-          {"No LLM API key found."}
-
-          <Link
-            href={`/project/${p.projectId}/settings`}
-            className="flex flex-row"
-          >
-            Create a new API key here. <ArrowTopRightIcon />
-          </Link>
-        </div>
-      ) : undefined}
-      <p className="text-sm text-muted-foreground">
-        The API key is used for each evaluation and will incur costs.
-      </p>
-    </div>
-  );
-};
-
 function getModelParamsWithEnabledFlag(
   evalPreFill?: EvalTemplateFormPreFill,
 ): UIModelParams {

web/src/ee/features/playground/page/context/index.tsx

@@ -14,6 +14,7 @@ import useCommandEnter from "@/src/ee/features/playground/page/hooks/useCommandE
 import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlaygroundCache";
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { extractVariables } from "@/src/utils/string";
 import {
   ChatMessageRole,
@@ -57,6 +58,7 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
   children,
 }) => {
   const capture = usePostHogClientCapture();
+  const projectId = useProjectIdFromURL();
   const { playgroundCache, setPlaygroundCache } = usePlaygroundCache();
   const [promptVariables, setPromptVariables] = useState<PromptVariable[]>([]);
   const [output, setOutput] = useState("");
@@ -172,6 +174,7 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
         }
 
         const completionStream = getChatCompletionStream(
+          projectId,
           finalMessages,
           modelParams,
         );
@@ -263,10 +266,17 @@ export const PlaygroundProvider: React.FC<PropsWithChildren> = ({
 };
 
 async function* getChatCompletionStream(
+  projectId: string | undefined,
   messages: ChatMessageWithId[],
   modelParams: UIModelParams,
 ) {
+  if (!projectId) {
+    console.error("Project ID is not set");
+    return;
+  }
+
   const body = JSON.stringify({
+    projectId,
     messages,
     modelParams: getFinalModelParams(modelParams),
   });

web/src/ee/features/playground/page/playground.tsx

@@ -16,7 +16,7 @@ export default function Playground() {
           <div className="basis-[55%] ">
             <ModelParameters {...playgroundContext} />
           </div>
-          <div className="basis-[45%]  overflow-auto">
+          <div className="mt-4 basis-[45%] overflow-auto">
             <Variables />
           </div>
         </div>

web/src/ee/features/playground/server/authorizeRequest.ts

@@ -0,0 +1,26 @@
+import { getServerSession } from "next-auth";
+
+import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
+import { getAuthOptions } from "@/src/server/auth";
+import { isProjectMemberOrAdmin } from "@/src/server/utils/checkProjectMembershipOrAdmin";
+import { ApiError, ForbiddenError, UnauthorizedError } from "@langfuse/shared";
+
+export type AuthorizeRequestResult = {
+  userId: string;
+};
+
+export const authorizeRequestOrThrow = async (
+  projectId: string,
+): Promise<AuthorizeRequestResult> => {
+  if (!getIsCloudEnvironment())
+    throw new ApiError("This endpoint is available in Langfuse cloud only.");
+
+  const authOptions = await getAuthOptions();
+  const session = await getServerSession(authOptions);
+  if (!session?.user) throw new UnauthorizedError("Unauthenticated");
+
+  if (!isProjectMemberOrAdmin(session.user, projectId))
+    throw new ForbiddenError("User is not a member of this project");
+
+  return { userId: session.user.id };
+};

web/src/ee/features/playground/server/chatCompletionHandler.ts

@@ -1,75 +1,60 @@
 import { StreamingTextResponse } from "ai";
-import { getToken } from "next-auth/jwt";
-import { type NextRequest, NextResponse } from "next/server";
+import { NextResponse, type NextRequest } from "next/server";
 
-import { fetchLLMCompletion } from "@langfuse/shared";
+import {
+  BaseError,
+  ValidationError,
+  fetchLLMCompletion,
+} from "@langfuse/shared";
 
 import { PosthogCallbackHandler } from "./analytics/posthogCallback";
-import {
-  validateChatCompletionBody,
-  type ValidatedChatCompletionBody,
-} from "./validateChatCompletionBody";
-import { getCookieName } from "@/src/server/utils/cookies";
-import { env } from "@/src/env.mjs";
-import { getIsCloudEnvironment } from "@/src/ee/utils/getIsCloudEnvironment";
+import { authorizeRequestOrThrow } from "./authorizeRequest";
+import { validateChatCompletionBody } from "./validateChatCompletionBody";
+
+import { prisma } from "@langfuse/shared/src/db";
+import { decrypt } from "@langfuse/shared/encryption";
 
 export default async function chatCompletionHandler(req: NextRequest) {
-  if (!getIsCloudEnvironment()) {
-    return NextResponse.json(
-      { message: "This endpoint is available in Langfuse cloud only." },
-      { status: 501 },
-    );
-  }
-
-  const token = await getToken({
-    req,
-    cookieName: getCookieName("next-auth.session-token"),
-    secret: env.NEXTAUTH_SECRET,
-  });
-
-  if (!token || !token.sub)
-    // sub is the user id
-    return NextResponse.json({ message: "Unauthenticated" }, { status: 401 });
-
-  if (req.method !== "POST")
-    return NextResponse.json(
-      { message: "Method not allowed" },
-      { status: 405 },
-    );
-
-  let body: ValidatedChatCompletionBody;
-
   try {
-    body = validateChatCompletionBody(await req.json());
-  } catch (err) {
-    console.error(err);
+    const body = validateChatCompletionBody(await req.json());
+    const { userId } = await authorizeRequestOrThrow(body.projectId);
 
-    return NextResponse.json(
-      {
-        message: "Invalid request body",
-        error: err,
-      },
-      { status: 400 },
-    );
-  }
-
-  try {
     const { messages, modelParams } = body;
+
+    const LLMApiKey = await prisma.llmApiKeys.findFirst({
+      where: {
+        projectId: body.projectId,
+        provider: modelParams.provider,
+      },
+    });
+
+    if (!LLMApiKey)
+      throw new ValidationError(
+        `No ${modelParams.provider} API key found in project. Please add one in the project settings.`,
+      );
+
     const stream = await fetchLLMCompletion({
       messages,
       modelParams,
       streaming: true,
-      callbacks: [new PosthogCallbackHandler("playground", body, token.sub)],
-      apiKey:
-        modelParams.provider === "openai"
-          ? env.OPENAI_API_KEY
-          : env.ANTHROPIC_API_KEY,
+      callbacks: [new PosthogCallbackHandler("playground", body, userId)],
+      apiKey: decrypt(LLMApiKey.secretKey),
     });
 
     return new StreamingTextResponse(stream);
   } catch (err) {
     console.error(err);
 
+    if (err instanceof BaseError) {
+      return NextResponse.json(
+        {
+          error: err.name,
+          message: err.message,
+        },
+        { status: err.httpCode },
+      );
+    }
+
     if (err instanceof Error) {
       return NextResponse.json(
         {

web/src/ee/features/playground/server/validateChatCompletionBody.ts

@@ -32,6 +32,7 @@ const MessageSchema = z.object({
   id: z.string().optional(),
 });
 export const ChatCompletionBodySchema = z.object({
+  projectId: z.string(),
   messages: z.array(MessageSchema),
   modelParams: ModelParamsSchema,
 });

web/src/env.mjs

@@ -67,6 +67,10 @@ export const env = createEnv({
     AUTH_AUTH0_CLIENT_SECRET: z.string().optional(),
     AUTH_AUTH0_ISSUER: z.string().url().optional(),
     AUTH_AUTH0_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_COGNITO_CLIENT_ID: z.string().optional(),
+    AUTH_COGNITO_CLIENT_SECRET: z.string().optional(),
+    AUTH_COGNITO_ISSUER: z.string().url().optional(),
+    AUTH_COGNITO_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT: z.string().optional(),
     AUTH_DISABLE_USERNAME_PASSWORD: z.enum(["true", "false"]).optional(),
     AUTH_DISABLE_SIGNUP: z.enum(["true", "false"]).optional(),
@@ -167,6 +171,10 @@ export const env = createEnv({
     AUTH_AUTH0_ISSUER: process.env.AUTH_AUTH0_ISSUER,
     AUTH_AUTH0_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
+    AUTH_COGNITO_CLIENT_ID: process.env.AUTH_COGNITO_CLIENT_ID,
+    AUTH_COGNITO_CLIENT_SECRET: process.env.AUTH_COGNITO_CLIENT_SECRET,
+    AUTH_COGNITO_ISSUER: process.env.AUTH_COGNITO_ISSUER,
+    AUTH_COGNITO_ALLOW_ACCOUNT_LINKING: process.env.AUTH_COGNITO_ALLOW_ACCOUNT_LINKING,
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT:
       process.env.AUTH_DOMAINS_WITH_SSO_ENFORCEMENT,
     AUTH_DISABLE_USERNAME_PASSWORD: process.env.AUTH_DISABLE_USERNAME_PASSWORD,

web/src/features/email/lib/project-invitation.ts

@@ -11,7 +11,7 @@ const langfuseUrls = {
   STAGING: "https://staging.langfuse.com",
 };
 
-const authUrl =
+const getAuthURL = () =>
   env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "US" ||
   env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "EU" ||
   env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
@@ -40,7 +40,7 @@ export const sendProjectInvitation = async (
         invitedByUserEmail: inviterEmail,
         projectName: projectName,
         recieverEmail: to,
-        inviteLink: authUrl,
+        inviteLink: getAuthURL(),
         langfuseCloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
       }),
     );

web/src/features/llm-api-key/types.ts

@@ -1,13 +1,8 @@
 import { z } from "zod";
 import { ModelProvider } from "@langfuse/shared";
 
-export const ZodModelProvider = z.enum([
-  ModelProvider.Anthropic,
-  ModelProvider.OpenAI,
-]);
-
 export const CreateLlmApiKey = z.object({
   projectId: z.string(),
   secretKey: z.string().min(1),
-  provider: z.literal(ModelProvider.OpenAI),
+  provider: z.nativeEnum(ModelProvider),
 });

web/src/features/manual-scoring/components/ManualScoreButton.tsx

@@ -77,13 +77,13 @@ export function ManualScoreButton({
       utils.sessions.invalidate(),
     ]);
   };
-  const mutCreateScore = api.scores.createReviewScore.useMutation({
+  const mutCreateScore = api.scores.createAnnotationScore.useMutation({
     onSuccess,
   });
-  const mutUpdateScore = api.scores.updateReviewScore.useMutation({
+  const mutUpdateScore = api.scores.updateAnnotationScore.useMutation({
     onSuccess,
   });
-  const mutDeleteScore = api.scores.deleteReviewScore.useMutation({
+  const mutDeleteScore = api.scores.deleteAnnotationScore.useMutation({
     onSuccess,
   });
 

web/src/features/posthog-analytics/usePostHogClientCapture.ts

@@ -91,6 +91,7 @@ const events = {
     "create_new_button_click",
   ],
   onboarding: ["code_example_tab_switch"],
+  user_settings: ["theme_changed"],
   project_settings: [
     "project_delete",
     "rename_form_submit",

web/src/features/public-api/components/LLMApiKeyList.tsx

@@ -175,7 +175,7 @@ function DeleteApiKeyButton(props: { projectId: string; apiKeyId: string }) {
 
 const formSchema = z.object({
   secretKey: z.string().min(1),
-  provider: z.literal(ModelProvider.OpenAI),
+  provider: z.nativeEnum(ModelProvider),
 });
 
 export function CreateLlmApiKeyComponent(props: {
@@ -277,7 +277,7 @@ export function CreateLlmApiKeyComponent(props: {
                     <Select
                       defaultValue={field.value}
                       onValueChange={(value) =>
-                        field.onChange(value as ModelProvider[number])
+                        field.onChange(value as ModelProvider)
                       }
                     >
                       <FormControl>
@@ -286,13 +286,7 @@ export function CreateLlmApiKeyComponent(props: {
                         </SelectTrigger>
                       </FormControl>
                       <SelectContent>
-                        {Array.from(
-                          new Set(
-                            evalLLMModels.map(
-                              (models) => models.provider.value,
-                            ),
-                          ),
-                        ).map((provider) => (
+                        {Object.values(ModelProvider).map((provider) => (
                           <SelectItem value={provider} key={provider}>
                             {provider}
                           </SelectItem>

web/src/features/tag/components/TagMananger.tsx

@@ -59,7 +59,7 @@ const TagManager = ({
   return (
     <Popover onOpenChange={(open) => handlePopoverChange(open)}>
       <PopoverTrigger className="select-none" asChild>
-        <div className="flex gap-x-2 gap-y-1">
+        <div className="flex flex-wrap gap-x-2 gap-y-1">
           <TagList selectedTags={selectedTags} isLoading={isLoading} />
         </div>
       </PopoverTrigger>

web/src/features/theming/ThemeToggle.tsx

@@ -2,10 +2,11 @@ import * as React from "react";
 import { Monitor, Moon, Sun } from "lucide-react";
 import { useTheme } from "next-themes";
 import { cn } from "@/src/utils/tailwind";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 
 export function ThemeToggle() {
   const { theme, setTheme } = useTheme();
-
+  const capture = usePostHogClientCapture();
   return (
     <div className="ml-auto flex items-center space-x-1">
       <div title="Light mode">
@@ -17,6 +18,9 @@ export function ThemeToggle() {
           onClick={(e) => {
             e.preventDefault();
             setTheme("light");
+            capture("user_settings:theme_changed", {
+              theme: "light",
+            });
           }}
         />
       </div>
@@ -29,6 +33,9 @@ export function ThemeToggle() {
           onClick={(e) => {
             e.preventDefault();
             setTheme("dark");
+            capture("user_settings:theme_changed", {
+              theme: "dark",
+            });
           }}
         />
       </div>
@@ -41,6 +48,9 @@ export function ThemeToggle() {
           onClick={(e) => {
             e.preventDefault();
             setTheme("system");
+            capture("user_settings:theme_changed", {
+              theme: "system",
+            });
           }}
         />
       </div>

web/src/pages/api/chatCompletion.ts

@@ -1,5 +0,0 @@
-import chatCompletionHandler from "@/src/ee/features/playground/server/chatCompletionHandler";
-
-export const runtime = "edge";
-
-export default chatCompletionHandler;

web/src/pages/api/public/ingestion.ts

@@ -258,15 +258,21 @@ const handleSingleEvent = async (
   req: NextApiRequest,
   apiScope: ApiAccessScope,
 ) => {
-  if ("body" in event && "input" in event.body && "output" in event.body) {
+  const { body } = event;
+  let restEvent = body;
+  if ("input" in body) {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const { input, output, ...restEvent } = event.body;
-    console.log(
-      `handling single event ${event.id} ${JSON.stringify({ event, body: restEvent })}`,
-    );
-  } else {
-    console.log(`handling single event ${event.id} ${JSON.stringify(event)}`);
+    const { input, ...rest } = body;
+    restEvent = rest;
   }
+  if ("output" in restEvent) {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { output, ...rest } = restEvent;
+    restEvent = rest;
+  }
+  console.log(
+    `handling single event ${event.id} ${JSON.stringify({ event, body: restEvent })}`,
+  );
 
   const cleanedEvent = ingestionEvent.parse(cleanEvent(event));
 

web/src/pages/auth/sign-in.tsx

@@ -14,7 +14,7 @@ import { env } from "@/src/env.mjs";
 import { zodResolver } from "@hookform/resolvers/zod";
 import { FcGoogle } from "react-icons/fc";
 import { FaGithub } from "react-icons/fa";
-import { SiOkta, SiAuth0 } from "react-icons/si";
+import { SiOkta, SiAuth0, SiAmazoncognito } from "react-icons/si";
 import { TbBrandAzure } from "react-icons/tb";
 import { signIn } from "next-auth/react";
 import Head from "next/head";
@@ -49,6 +49,7 @@ export type PageProps = {
     okta: boolean;
     azureAd: boolean;
     auth0: boolean;
+    cognito: boolean;
     sso: boolean;
   };
   signUpDisabled: boolean;
@@ -80,6 +81,10 @@ export const getServerSideProps: GetServerSideProps<PageProps> = async () => {
           env.AUTH_AUTH0_CLIENT_ID !== undefined &&
           env.AUTH_AUTH0_CLIENT_SECRET !== undefined &&
           env.AUTH_AUTH0_ISSUER !== undefined,
+        cognito:
+          env.AUTH_COGNITO_CLIENT_ID !== undefined &&
+          env.AUTH_COGNITO_CLIENT_SECRET !== undefined &&
+          env.AUTH_COGNITO_ISSUER !== undefined,
         sso,
       },
       signUpDisabled: env.AUTH_DISABLE_SIGNUP === "true",
@@ -169,6 +174,18 @@ export function SSOButtons({
               Auth0
             </Button>
           )}
+          {authProviders.cognito && (
+            <Button
+              onClick={() => {
+                capture("sign_in:button_click", { provider: "cognito" });
+                void signIn("cognito");
+              }}
+              variant="secondary"
+            >
+              <SiAmazoncognito className="mr-3" size={18} />
+              Cognito
+            </Button>
+          )}
         </div>
       </div>
     ) : null
@@ -421,7 +438,7 @@ export default function SignIn({ authProviders, signUpDisabled }: PageProps) {
               No account yet?{" "}
               <Link
                 href="/auth/sign-up"
-                className="hover:text-hover-primary-accent font-semibold leading-6 text-primary-accent"
+                className="font-semibold leading-6 text-primary-accent hover:text-hover-primary-accent"
               >
                 Sign up
               </Link>

web/src/server/api/definitions/scoresTable.ts

@@ -1,6 +1,7 @@
 import {
   type OptionsDefinition,
   type ColumnDefinition,
+  ScoreSource,
 } from "@langfuse/shared";
 
 export const scoresTableCols: ColumnDefinition[] = [
@@ -31,8 +32,9 @@ export const scoresTableCols: ColumnDefinition[] = [
   {
     name: "Source",
     id: "source",
-    type: "string",
-    internal: 's."source"',
+    type: "stringOptions",
+    internal: 's."source"::text',
+    options: Object.values(ScoreSource).map((value) => ({ value })),
   },
   {
     name: "Name",

web/src/server/api/routers/scores.ts

@@ -121,7 +121,7 @@ export const scoresRouter = createTRPCRouter({
 
       return res;
     }),
-  createReviewScore: protectedProjectProcedure
+  createAnnotationScore: protectedProjectProcedure
     .input(
       z.object({
         projectId: z.string(),
@@ -158,7 +158,7 @@ export const scoresRouter = createTRPCRouter({
           name: input.name,
           comment: input.comment,
           authorUserId: ctx.session.user.id,
-          source: "REVIEW",
+          source: "ANNOTATION",
         },
       });
       await auditLog({
@@ -174,7 +174,7 @@ export const scoresRouter = createTRPCRouter({
       });
       return score;
     }),
-  updateReviewScore: protectedProjectProcedure
+  updateAnnotationScore: protectedProjectProcedure
     .input(
       z.object({
         projectId: z.string(),
@@ -193,11 +193,11 @@ export const scoresRouter = createTRPCRouter({
         where: {
           id: input.id,
           projectId: input.projectId,
-          source: "REVIEW",
+          source: "ANNOTATION",
         },
       });
       if (!score) {
-        throw new Error("No review score with this id in this project.");
+        throw new Error("No annotation score with this id in this project.");
       }
 
       await auditLog({
@@ -224,7 +224,7 @@ export const scoresRouter = createTRPCRouter({
         },
       });
     }),
-  deleteReviewScore: protectedProjectProcedure
+  deleteAnnotationScore: protectedProjectProcedure
     .input(z.object({ projectId: z.string(), id: z.string() }))
     .mutation(async ({ input, ctx }) => {
       throwIfNoAccess({
@@ -236,12 +236,12 @@ export const scoresRouter = createTRPCRouter({
       const score = await ctx.prisma.score.findFirst({
         where: {
           id: input.id,
-          source: "REVIEW",
+          source: "ANNOTATION",
           projectId: input.projectId,
         },
       });
       if (!score) {
-        throw new Error("No review score with this id in this project.");
+        throw new Error("No annotation score with this id in this project.");
       }
 
       await auditLog({

web/src/server/api/trpc.ts

@@ -79,6 +79,7 @@ import superjson from "superjson";
 import { ZodError } from "zod";
 import { setUpSuperjson } from "@/src/utils/superjson";
 import { DB } from "@/src/server/db";
+import { isProjectMemberOrAdmin } from "@/src/server/utils/checkProjectMembershipOrAdmin";
 
 setUpSuperjson();
 
@@ -177,7 +178,7 @@ const enforceUserIsAuthedAndProjectMember = t.middleware(
       ({ id }) => id === projectId,
     );
 
-    if (!sessionProject && ctx.session.user.admin !== true)
+    if (!sessionProject && !isProjectMemberOrAdmin(ctx.session.user, projectId))
       throw new TRPCError({
         code: "UNAUTHORIZED",
         message: "User is not a member of this project",
@@ -242,7 +243,11 @@ const enforceTraceAccess = t.middleware(async ({ ctx, rawInput, next }) => {
     ({ id }) => id === trace.projectId,
   );
 
-  if (!trace.public && !sessionProject && ctx.session?.user?.admin !== true)
+  if (
+    !trace.public &&
+    !sessionProject &&
+    !isProjectMemberOrAdmin(ctx.session?.user, trace.projectId)
+  )
     throw new TRPCError({
       code: "UNAUTHORIZED",
       message:

web/src/server/auth.ts

@@ -19,9 +19,10 @@ import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
 import OktaProvider from "next-auth/providers/okta";
 import Auth0Provider from "next-auth/providers/auth0";
+import CognitoProvider from "next-auth/providers/cognito";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { type Provider } from "next-auth/providers/index";
-import { getCookieName, cookieOptions } from "./utils/cookies";
+import { getCookieName, getCookieOptions } from "./utils/cookies";
 import {
   getSsoAuthProviderIdForDomain,
   loadSsoProviders,
@@ -182,6 +183,21 @@ if (
     }),
   );
 
+if (
+  env.AUTH_COGNITO_CLIENT_ID &&
+  env.AUTH_COGNITO_CLIENT_SECRET &&
+  env.AUTH_COGNITO_ISSUER
+)
+  staticProviders.push(
+    CognitoProvider({
+      clientId: env.AUTH_COGNITO_CLIENT_ID,
+      clientSecret: env.AUTH_COGNITO_CLIENT_SECRET,
+      issuer: env.AUTH_COGNITO_ISSUER,
+      allowDangerousEmailAccountLinking:
+        env.AUTH_COGNITO_ALLOW_ACCOUNT_LINKING === "true",
+    }),
+  );
+
 // Extend Prisma Adapter
 const prismaAdapter = PrismaAdapter(prisma);
 const extendedPrismaAdapter: Adapter = {
@@ -314,27 +330,27 @@ export async function getAuthOptions(): Promise<NextAuthOptions> {
     cookies: {
       sessionToken: {
         name: getCookieName("next-auth.session-token"),
-        options: cookieOptions,
+        options: getCookieOptions(),
       },
       csrfToken: {
         name: getCookieName("next-auth.csrf-token"),
-        options: cookieOptions,
+        options: getCookieOptions(),
       },
       callbackUrl: {
         name: getCookieName("next-auth.callback-url"),
-        options: cookieOptions,
+        options: getCookieOptions(),
       },
       state: {
         name: getCookieName("next-auth.state"),
-        options: cookieOptions,
+        options: getCookieOptions(),
       },
       nonce: {
         name: getCookieName("next-auth.nonce"),
-        options: cookieOptions,
+        options: getCookieOptions(),
       },
       pkceCodeVerifier: {
         name: getCookieName("next-auth.pkce.code_verifier"),
-        options: cookieOptions,
+        options: getCookieOptions(),
       },
     },
     events: {

web/src/server/utils/checkProjectMembershipOrAdmin.ts

@@ -0,0 +1,15 @@
+import { type User } from "next-auth";
+
+export const isProjectMemberOrAdmin = (
+  user: User | null | undefined,
+  projectId: string,
+): boolean => {
+  if (!user) return false;
+
+  const isAdmin = user.admin === true;
+  const isProjectMember = user.projects.some(
+    (project) => project.id === projectId,
+  );
+
+  return isProjectMember || isAdmin;
+};

web/src/server/utils/cookies.ts

@@ -1,20 +1,20 @@
 import { env } from "@/src/env.mjs";
 
 // Use secure cookies on https hostnames, exception for Vercel which sets NEXTAUTH_URL without the protocol
-const shouldSecureCookies =
+const shouldSecureCookies = () =>
   env.NEXTAUTH_URL.startsWith("https://") || process.env.VERCEL === "1";
 
-export const cookieOptions = {
+export const getCookieOptions = () => ({
   domain: env.NEXTAUTH_COOKIE_DOMAIN ?? undefined,
   httpOnly: true,
   sameSite: "lax" as const,
   path: "/",
-  secure: shouldSecureCookies,
-};
+  secure: shouldSecureCookies(),
+});
 
 export const getCookieName = (name: string) =>
   [
-    shouldSecureCookies ? "__Secure-" : "",
+    shouldSecureCookies() ? "__Secure-" : "",
     name,
     env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
       ? `.${env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}`

worker/generated/types.ts

@@ -28,11 +28,16 @@ export const ObservationLevel = {
 export type ObservationLevel =
   (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
+  ANNOTATION: "ANNOTATION",
   API: "API",
-  REVIEW: "REVIEW",
   EVAL: "EVAL",
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
+export const ScoreDataType = {
+  CATEGORICAL: "CATEGORICAL",
+  NUMERIC: "NUMERIC",
+} as const;
+export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
 export const PricingUnit = {
   PER_1000_TOKENS: "PER_1000_TOKENS",
   PER_1000_CHARS: "PER_1000_CHARS",
@@ -356,6 +361,22 @@ export type Score = {
   comment: string | null;
   trace_id: string;
   observation_id: string | null;
+  config_id: string | null;
+  string_value: string | null;
+  data_type: Generated<ScoreDataType>;
+};
+export type ScoreConfig = {
+  id: string;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+  project_id: string;
+  name: string;
+  data_type: ScoreDataType;
+  is_archived: Generated<boolean>;
+  min_value: number | null;
+  max_value: number | null;
+  categories: unknown | null;
+  description: string | null;
 };
 export type Session = {
   id: string;
@@ -455,6 +476,7 @@ export type DB = {
   project_memberships: ProjectMembership;
   projects: Project;
   prompts: Prompt;
+  score_configs: ScoreConfig;
   scores: Score;
   Session: Session;
   sso_configs: SsoConfig;

worker/package.json

@@ -1,6 +1,6 @@
 {
   "name": "worker",
-  "version": "2.45.0",
+  "version": "2.46.0",
   "description": "",
   "license": "MIT",
   "main": "index.js",

worker/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.45.0";
+export const VERSION = "v2.46.0";