chore: release v2.49.1

* chore: update MainNavigation component to use nested navigation literals

* simplify

---------

Co-authored-by: Marc Klingen <git@marcklingen.com>

.env.dev.example

@@ -47,5 +47,5 @@ REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
 LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
-# openssl rand -base64 32 used only here
+# openssl rand -hex 32 used only here
 ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 

.env.prod.example

@@ -58,6 +58,11 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # AUTH_AUTH0_CLIENT_SECRET=
 # AUTH_AUTH0_ISSUER=
 # AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false
+# AUTH_COGNITO_CLIENT_ID=
+# AUTH_COGNITO_CLIENT_SECRET=
+# AUTH_COGNITO_ISSUER=
+# AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -80,9 +85,23 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # ENABLE_EVENT_LOG=false
 
 
+
+### START Envs to be deprecated in Langfuse v3.0
+
+# Disable the expensive analytics queries and related features
+# LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
+
+# Customize the default datetime offset filter used in for the UI tables
+# LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET="7" # int (days)
+
+### END Envs to be deprecated in Langfuse v3.0
+
+
+
+
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
-# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
+# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMANTIC VERSIONING
 
 # NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
 # NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -12,10 +12,24 @@ body:
   - type: textarea
     attributes:
       label: To reproduce
-      description: Describe how to reproduce your bug. Steps, code snippets, reproduction repos etc.
+      description: Describe how to reproduce your bug. Please provide detailed steps, code snippets, reproduction repos etc.
     validations:
       required: true
+  - type: textarea
+    attributes:
+      label: SDK and container versions
+      description: If you're experiencing an issue with an integration or SDK, please ensure you're using the latest version. If you're self-hosting Langfuse, check that you're running the most recent version. If updating isn't an option, please provide the specific versions you're currently using.
   - type: textarea
     attributes:
       label: Additional information
       description: Add any other information related to the bug here, screenshots if applicable.
+  - type: dropdown
+    id: contribute
+    attributes:
+      label: Are you interested to contribute a fix for this bug?
+      description: If this is a confirmed bug, the maintainers are happy to support with guidance and review.
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true

.github/workflows/pipeline.yml

@@ -40,19 +40,27 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Build Web Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./web/Dockerfile
-          push: false
+      - name: Build and run both images from compose
+        run: |
+          docker compose -f docker-compose.build.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
 
-      - name: Build Worker Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./worker/Dockerfile
-          push: false
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
+
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+
+      - name: Ensure no unhealthy status
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
 
   tests-web:
     runs-on: ubuntu-latest
@@ -109,9 +117,6 @@ jobs:
     strategy:
       matrix:
         node-version: [20]
-    env:
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }}
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master

.github/workflows/porter_stack_worker.yml

@@ -0,0 +1,29 @@
+"on":
+  push:
+    branches:
+    - main
+name: Deploy to worker
+jobs:
+  porter-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Set Github tag
+      id: vars
+      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+    - name: Setup porter
+      uses: porter-dev/setup-porter@v0.1.0
+    - name: Deploy stack
+      timeout-minutes: 30
+      run: exec porter apply
+      env:
+        PORTER_CLUSTER: "3959"
+        PORTER_DEPLOYMENT_TARGET_ID: d2bb23e4-1d77-48f2-a113-383d076959b3
+        PORTER_HOST: https://dashboard.getporter.dev
+        PORTER_PR_NUMBER: ${{ github.event.number }}
+        PORTER_PROJECT: "12565"
+        PORTER_REPO_NAME: ${{ github.event.repository.name }}
+        PORTER_STACK_NAME: worker
+        PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+        PORTER_TOKEN: ${{ secrets.PORTER_STACK_12565_3959 }}

CONTRIBUTING.md

@@ -191,6 +191,16 @@ Requirements
     pnpm run dev
    ```
 
+7. Open the web app in the browser:
+
+   http://localhost:3000
+
+8. Log in as a test user (after you ran `db:seed` command):
+
+   Username: demo@langfuse.com
+
+   Password: password
+
 ## Monorepo quickstart
 
 - Available packages and their dependencies
@@ -290,6 +300,86 @@ When a new release is tagged on the `main` branch (excluding prereleases), it tr
 1. The Docker image is published to GitHub Packages with the version number and `latest` tag.
 2. The deployment is carried out on Langfuse Cloud. This is done by force pushing the `main` branch to the `production` branch during every release, using the [`release.yml`](.github/workflows/release.yml) GitHub Action.
 
+## Theming
+
+At Langfuse, we utilize CSS variables to manage our theme settings across the platform.
+
+Our approach leverages separate CSS variables for backgrounds (--background) and foregrounds (--foreground), fully adhering to the [shadcn/ui](https://ui.shadcn.com/docs/theming) color conventions. The background suffix can be omitted if the variable is used for the background color of the component. We recommend using HSL values for these colors to enhance consistency and customization. There is no need to manually handle dark mode styling with "dark:" prefixes, as next-themes automatically manages the theme switching.
+
+Given the following CSS variables:
+
+```
+--primary: 222.2 47.4% 11.2%; // e.g. background-color
+--primary-foreground: 210 40% 98%; // e.g. text-color
+```
+
+The background color of the following component will be `hsl(var(--primary))` and the foreground color will be `hsl(var(--primary-foreground))`.
+
+```
+<div class="bg-primary text-primary-foreground">Hello</div>
+```
+
+### Color Variables
+
+| Variable                 | Description                                                        | Examples                         |
+| ------------------------ | ------------------------------------------------------------------ | -------------------------------- |
+| --background             | Background color                                                   | Default background color of body |
+| --foreground             | Foreground color                                                   | Default text color of body       |
+| --muted                  | Muted background color                                             | TabsList, Skeleton and Switch    |
+| --muted-foreground       | Muted foreground color                                             |                                  |
+| --popover                | Popover background color                                           | DropdownMenu, HoverCard, Popover |
+| --popover-foreground     | Popover foreground color                                           |                                  |
+| --card                   | Card background color                                              | Card                             |
+| --card-foreground        | Card foreground color                                              |                                  |
+| --border                 | Border color                                                       | Default border color             |
+| --input                  | Input field border color                                           | Input, Select, Textarea          |
+| --primary                | Primary button background colors                                   | Button variant="primary"         |
+| --primary-foreground     | Primary button foreground color                                    |                                  |
+| --secondary              | Secondary button background color                                  | Button variant="secondary"       |
+| --secondary-foreground   | Secondary button foreground color                                  |                                  |
+| --accent                 | Used for accents such as hover effects                             | DropdownMenuItem, SelectItem     |
+| --accent-foreground      | Used for texts on hover effects                                    | DropdownMenuItem, SelectItem     |
+| --destructive            | Destructive action color for background                            | Button variant="destructive"     |
+| --destructive-foreground | Destructive action color for text                                  |                                  |
+| --ring                   | Focus ring color                                                   | MultiSelect                      |
+| --primary-accent         | Primary accent color used for branding                             | Layout                           |
+| --hover-primary-accent   | Primary accent color used for hover effects for links              | SignIn and AuthCloudRegionSwitch |
+| --muted-green            | Muted green for Event label                                        | ObservationTree                  |
+| --muted-orange           | Muted orange for Generation label                                  | ObservationTree                  |
+| --muted-blue             | Muted blue for Span label                                          | ObservationTree                  |
+| --muted-gray             | Muted gray for disabled status badges                              | StatusBadge                      |
+| --accent-light-green     | Light green accent for background of output and assistant messages | IOPreview, Generations, Traces   |
+| --accent-dark-green      | Dark green accent for border of output and assistant messages      | CodeJsonViewer and IOPReview     |
+| --light-red              | Light red for error background                                     | level-color and StatusBadge      |
+| --dark-red               | Dark red for error text and error badge dot color                  | level-color and ErrorPage        |
+| --light-yellow           | Light yellow for warning background                                | LevelColor                       |
+| --dark-yellow            | Dark yellow for warning text                                       | LevelColor                       |
+| --light-green            | Light green for success status badge background                    | StatusBadge                      |
+| --dark-green             | Dark green for success status badge text and dot                   | StatusBadge                      |
+| --light-blue             | Light blue for background of Staging label                         | LangfuseLogo                     |
+| --dark-blue              | Dark blue for text and border of Staging label                     | LangfuseLogo                     |
+| --accent-light-blue      | Light blue accent for table link hover effect                      | TableLink                        |
+| --accent-dark-blue       | Dark blue accent for table link text                               | TableLink                        |
+
+### Adding New Colors
+
+1. Global Definitions: Add new CSS variable definitions in the global.css file.
+2. Tailwind Configuration: Reflect these new colors in the tailwind.config.js to maintain alignment with Tailwind's utility classes.
+
+By following these guidelines, you can ensure that any contributions to our theme are consistent, maintainable, and aligned with our design system.
+
+## Maintainers
+
+### Using secrets stored in 1Password
+
+When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.
+
+Example:
+
+```bash
+op run --env-file="./.env" -- pnpm --filter=shared run db:deploy
+```
+
 ## License
 
 Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2023 Finto Technologies GmbH
+Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 

README.md

@@ -106,22 +106,26 @@ See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Lan
 
 ### Integrations
 
-| Integration                                                      | Supports      | Description                                                 |
-| ---------------------------------------------------------------- | ------------- | ----------------------------------------------------------- |
-| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
-| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python, JS/TS | Automated instrumentation of OpenAI SDK.                    |
-| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.                    |
-| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python        | Automated instrumentation via LlamaIndex callback system.   |
-| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.       |
+| Integration                                              | Supports                   | Description                                                                                                                                      |
+| -------------------------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [SDK](/docs/sdk)                                         | Python, JS/TS              | Manual instrumentation using the SDKs for full flexibility.                                                                                      |
+| [OpenAI](/docs/integrations/openai)                      | Python, JS/TS              | Automated instrumentation using drop-in replacement of OpenAI SDK.                                                                               |
+| [Langchain](/docs/integrations/langchain)                | Python, JS/TS              | Automated instrumentation by passing callback handler to Langchain application.                                                                  |
+| [LlamaIndex](/docs/integrations/llama-index/get-started) | Python                     | Automated instrumentation via LlamaIndex callback system.                                                                                        |
+| [Haystack](/docs/integrations/haystack)                  | Python                     | Automated instrumentation via Haystack content tracing system.                                                                                   |
+| [LiteLLM](/docs/integrations/litellm)                    | Python, JS/TS (proxy only) | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
+| [API](/docs/api)                                         |                            | Directly call the public API. OpenAPI spec available.                                                                                            |
 
-External projects/packages that integrate with Langfuse:
+Packages that integrate with Langfuse:
 
-| Name                                                            | Description                                                                                                                                      |
-| --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [LiteLLM](https://langfuse.com/docs/integrations/litellm)       | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
-| [Flowise](https://langfuse.com/docs/integrations/flowise)       | JS/TS no-code builder for customized LLM flows.                                                                                                  |
-| [Langflow](https://langfuse.com/docs/integrations/langflow)     | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows.                          |
-| [Superagent](https://langfuse.com/docs/integrations/superagent) | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                                               |
+| Name                                                       | Description                                                                                                             |
+| ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| [Instructor](/docs/integrations/instructor)                | Library to get structured LLM outputs (JSON, Pydantic)                                                                  |
+| [Mirascope](/docs/integrations/mirascope)                  | Python toolkit for building LLM applications.                                                                           |
+| [AI SDK by Vercel](/docs/sdk/typescript/example-vercel-ai) | Typescript SDK that makes streaming LLM outputs super easy.                                                             |
+| [Flowise](/docs/integrations/flowise)                      | JS/TS no-code builder for customized LLM flows.                                                                         |
+| [Langflow](/docs/integrations/langflow)                    | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
+| [Superagent](/docs/integrations/superagent)                | Open Source AI Assistant Framework & API for prototyping and deployment of agents.                                      |
 
 ## Questions and feedback
 

careers

@@ -0,0 +1,14 @@
+_We are Hiring_
+Join us in scaling Langfuse in Berlin, Germany. We are an open source company, we hire in person, we are only hiring technical talent.
+
+_Open Roles_
+
+- Backend Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mnrdwla-backend-engineer
+- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+
+_More Info_
+
+- https://langfuse.com/careers
+- https://langfuse.com/docs
+- https://langfuse.com/changelog

docker-compose.build.yml

@@ -20,6 +20,11 @@ services:
       - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
       - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   worker:
     build:
@@ -41,6 +46,11 @@ services:
       - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
       - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   redis:
     image: redis:7.2.4

docker-compose.yml

@@ -4,7 +4,8 @@ services:
   langfuse-server:
     image: ghcr.io/langfuse/langfuse:latest
     depends_on:
-      - db
+      db:
+        condition: service_healthy
     ports:
       - "3000:3000"
     environment:
@@ -18,6 +19,11 @@ services:
   db:
     image: postgres
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres

ee/LICENSE

@@ -0,0 +1,37 @@
+Langfuse Enterprise License (the “Enterprise License”)
+Copyright (c) 2024 Langfuse GmbH ('Langfuse')
+
+With regard to the Langfuse Software:
+This software and associated documentation files (the "Software") may only be
+used in production, if you (and any entity that you represent) have agreed to,
+and are in compliance with, the Langfuse Terms of Service, available
+at https://langfuse.com/terms (the “Enterprise Terms”), or other
+agreement governing the use of the Software, as agreed by you and Langfuse,
+and otherwise have a valid Langfuse Enterprise license.
+
+Subject to the foregoing sentence, you are free to
+modify this Software and publish patches to the Software. You agree that Langfuse
+and/or its licensors (as applicable) retain all right, title and interest in and
+to all such modifications and/or patches, and all such modifications and/or
+patches may only be used, copied, modified, displayed, distributed, or otherwise
+exploited with a valid Langfuse Enterprise license.
+
+Notwithstanding the foregoing, you may copy and modify
+the Software for development and testing purposes, without requiring a
+subscription. You agree that Langfuse and/or its licensors (as applicable) retain
+all right, title and interest in and to all such modifications. You are not
+granted any other rights beyond what is expressly stated herein. Subject to the
+foregoing, it is forbidden to copy, merge, publish, distribute, sublicense,
+and/or sell the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+For all third party components incorporated into the Langfuse Software, those
+components are licensed under the original license provided by the owner of the 
+applicable component.

ee/package.json

@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@langfuse/shared": "workspace:*",
-    "next": "^14.2.1",
+    "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "zod": "^3.22.4"
   },
@@ -33,7 +33,7 @@
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
     "@types/node": "^20.11.29",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",

ee/src/sso/index.ts

@@ -2,10 +2,11 @@ import { type Provider } from "next-auth/providers/index";
 import GoogleProvider from "next-auth/providers/google";
 import GitHubProvider from "next-auth/providers/github";
 import OktaProvider from "next-auth/providers/okta";
+import CognitoProvider from "next-auth/providers/cognito";
 import Auth0Provider from "next-auth/providers/auth0";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { isEeAvailable } from "..";
-import { prisma } from "@langfuse/shared/src/db";
+import { type SsoConfig, prisma } from "@langfuse/shared/src/db";
 import { encrypt, decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import { type NextApiRequest, type NextApiResponse } from "next";
@@ -25,14 +26,36 @@ let cachedSsoConfigs: {
 async function getSsoConfigs(): Promise<SsoProviderSchema[]> {
   if (!isEeAvailable) return [];
   const CACHE_TTL = 60 * 1000; // 1 minute
+  const DB_MAX_WAIT = 2000; // 2 seconds
+  const DB_TIMEOUT = 3000; // 3 seconds
 
   // Set/refresh the cache if it's empty or expired
   if (
     cachedSsoConfigs === null ||
     Date.now() - cachedSsoConfigs.timestamp > CACHE_TTL
   ) {
+    // findMany with custom timeout via $transaction
+    let dbConfigs: SsoConfig[] = [];
+    try {
+      dbConfigs = await prisma.$transaction(
+        async (prisma) => prisma.ssoConfig.findMany(),
+        {
+          maxWait: DB_MAX_WAIT,
+          timeout: DB_TIMEOUT,
+        }
+      );
+    } catch (e) {
+      // cache empty array to prevent repeated DB calls on error
+      cachedSsoConfigs = {
+        data: [],
+        timestamp: Date.now(),
+      };
+
+      // caught and logged in the caller
+      throw e;
+    }
+
     // transform into zod object
-    const dbConfigs = await prisma.ssoConfig.findMany();
     const parsedSsoConfigs = dbConfigs
       .map((v) => {
         try {
@@ -144,6 +167,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "cognito")
+    return CognitoProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line no-unused-vars

ee/src/sso/types.ts

@@ -64,11 +64,24 @@ export const AzureAdProviderSchema = base.extend({
     .nullish(),
 });
 
+export const CognitoProviderSchema = base.extend({
+  authProvider: z.literal("cognito"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
@@ -76,6 +89,7 @@ export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   Auth0ProviderSchema,
   OktaProviderSchema,
   AzureAdProviderSchema,
+  CognitoProviderSchema,
 ]);
 
 export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

fern/apis/client/generators.yml

@@ -15,7 +15,7 @@ groups:
         version: 0.0.26
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-client
+          path: ../../../web/public/generated/api-client
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true

fern/apis/server/definition/api.yml

@@ -9,8 +9,8 @@ docs: |
 
   ## Exports
 
-  - OpenAPI spec: https://cloud.langfuse.com/openapi-server.yml
-  - Postman collection: https://cloud.langfuse.com/postman-collection.json
+  - OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
+  - Postman collection: https://cloud.langfuse.com/generated/postman/collection.json
 
 error-discrimination:
   strategy: status-code

fern/apis/server/definition/commons.yml

@@ -209,8 +209,8 @@ types:
       - ARCHIVED
   ScoreSource:
     enum:
+      - ANNOTATION
       - API
-      - REVIEW
       - EVAL
 
 errors:

fern/apis/server/definition/dataset-items.yml

@@ -32,3 +32,6 @@ types:
       id:
         type: optional<string>
         docs: Dataset items are upserted on their id
+      status:
+        type: optional<commons.DatasetStatus>
+        docs: Defaults to ACTIVE for newly created items

fern/apis/server/definition/metrics.yml

@@ -31,10 +31,10 @@ service:
             docs: Optional filter for metrics where traces include all of these tags
           fromTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces on or after a certain timestamp
+            docs: Optional filter to only include traces on or after a certain datetime (ISO 8601)
           toTimestamp:
             type: optional<datetime>
-            docs: Optional filter to only include traces before a certain timestamp
+            docs: Optional filter to only include traces before a certain datetime (ISO 8601)
       response: DailyMetrics
 types:
   DailyMetrics:
@@ -48,12 +48,25 @@ types:
       date: date
       countTraces: integer
       countObservations: integer
-      totalCost: double
+      totalCost:
+        type: double
+        docs: Total model cost in USD
       usage: list<UsageByModel>
   UsageByModel:
     docs: Daily usage of a given model. Usage corresponds to the unit set for the specific model (e.g. tokens).
     properties:
       model: optional<string>
-      inputUsage: integer
-      outputUsage: integer
-      totalUsage: integer
+      inputUsage:
+        type: integer
+        docs: Total number of generation input units (e.g. tokens)
+      outputUsage:
+        type: integer
+        docs: Total number of generation output units (e.g. tokens)
+      totalUsage:
+        type: integer
+        docs: Total number of generation total units (e.g. tokens)
+      countTraces: integer
+      countObservations: integer
+      totalCost:
+        type: double
+        docs: Total model cost in USD

fern/apis/server/definition/observations.yml

@@ -35,7 +35,7 @@ service:
           parentObservationId: optional<string>
           fromStartTime:
             type: optional<datetime>
-            docs: Retrieve only observations with a start_time greater than this timestamp.
+            docs: Retrieve only observations with a start_time greater than this datetime (ISO 8601).
       response: ObservationsViews
 
 types:

fern/apis/server/definition/prompts.yml

@@ -70,14 +70,24 @@ types:
       name: string
       prompt: list<ChatMessage>
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   CreateTextPromptRequest:
     properties:
       name: string
       prompt: string
       config: optional<unknown>
-      labels: optional<list<string>>
+      labels:
+        type: optional<list<string>>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: optional<list<string>>
+        docs: List of tags to apply to all versions of this prompt.
 
   Prompt:
     union:
@@ -89,7 +99,12 @@ types:
       name: string
       version: integer
       config: unknown
-      labels: list<string>
+      labels:
+        type: list<string>
+        docs: List of deployment labels of this prompt version.
+      tags:
+        type: list<string>
+        docs: List of tags. Used to filter via UI and API. The same across versions of a prompt.
 
   ChatMessage:
     properties:

fern/apis/server/definition/score.yml

@@ -29,7 +29,7 @@ service:
           name: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only scores newer than this timestamp.
+            docs: Retrieve only scores newer than this datetime (ISO 8601).
           source:
             type: commons.ScoreSource
             docs: Retrieve only scores from a specific source.

fern/apis/server/definition/sessions.yml

@@ -7,7 +7,7 @@ service:
   base-path: /api/public
   endpoints:
     get:
-      docs: Get a session
+      docs: Get a session. Please note that `traces` on this endpoint are not paginated, if you plan to fetch large sessions, consider `GET /api/public/traces?sessionId=<sessionId>`
       method: GET
       path: /sessions/{sessionId}
       path-parameters:

fern/apis/server/definition/trace.yml

@@ -30,9 +30,10 @@ service:
             docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           userId: optional<string>
           name: optional<string>
+          sessionId: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only traces newer than this timestamp.
+            docs: Retrieve only traces newer than this datetime (ISO 8601).
           orderBy:
             type: optional<string>
             docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"

fern/apis/server/generators.yml

@@ -6,7 +6,7 @@ groups:
         version: 0.0.28
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-server
+          path: ../../../web/public/generated/api
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true
@@ -33,7 +33,7 @@ groups:
         version: 0.0.45
         output:
           location: local-file-system
-          path: ../../../web/generated/postman
+          path: ../../../web/public/generated/postman
   # published:
   #   generators:
   #     - name: fernapi/fern-python-sdk

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.37.2",
+  "version": "2.49.1",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -28,10 +28,10 @@
   },
   "devDependencies": {
     "@release-it/bumper": "^6.0.1",
-    "dotenv-cli": "^7.4.1",
+    "dotenv-cli": "^7.4.2",
     "prettier": "^3.2.5",
-    "release-it": "^17.2.0",
-    "turbo": "^1.13.2"
+    "release-it": "^17.2.1",
+    "turbo": "^1.13.3"
   },
   "release-it": {
     "git": {

packages/config-eslint/package.json

@@ -9,11 +9,11 @@
   ],
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "@vercel/style-guide": "^6.0.0",
-    "eslint-config-next": "^14.2.1",
+    "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.2",
+    "eslint-config-turbo": "^1.13.3",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/package.json

@@ -44,14 +44,15 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@langchain/anthropic": "^0.1.14",
-    "@langchain/core": "^0.1.58",
-    "@langchain/openai": "^0.0.28",
-    "@prisma/client": "^5.12.1",
+    "@langchain/anthropic": "^0.1.21",
+    "@langchain/core": "^0.1.61",
+    "@langchain/openai": "^0.0.33",
+    "@prisma/client": "^5.13.0",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.33",
+    "langchain": "^0.1.36",
+    "lodash": "^4.17.21",
     "prisma-extension-kysely": "^2.1.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.5"
@@ -59,11 +60,11 @@
   "devDependencies": {
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
-    "@types/lodash": "^4.14.202",
+    "@types/lodash": "^4.17.4",
     "@types/node": "^20.11.29",
-    "@types/pg": "^8.11.5",
+    "@types/pg": "^8.11.6",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
@@ -71,12 +72,12 @@
     "kysely-codegen": "^0.11.0",
     "nodemon": "^3.0.3",
     "prettier": "^3.2.5",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
-    "vitest": "^1.5.0"
+    "vitest": "^1.5.3"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -4,13 +4,13 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
-export const MembershipRole = {
+export const ProjectRole = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
     MEMBER: "MEMBER",
     VIEWER: "VIEWER"
 } as const;
-export type MembershipRole = (typeof MembershipRole)[keyof typeof MembershipRole];
+export type ProjectRole = (typeof ProjectRole)[keyof typeof ProjectRole];
 export const ObservationType = {
     SPAN: "SPAN",
     EVENT: "EVENT",
@@ -25,22 +25,17 @@ export const ObservationLevel = {
 } as const;
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
+    ANNOTATION: "ANNOTATION",
     API: "API",
-    REVIEW: "REVIEW",
     EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
-export const PricingUnit = {
-    PER_1000_TOKENS: "PER_1000_TOKENS",
-    PER_1000_CHARS: "PER_1000_CHARS"
+export const ScoreDataType = {
+    CATEGORICAL: "CATEGORICAL",
+    NUMERIC: "NUMERIC",
+    BOOLEAN: "BOOLEAN"
 } as const;
-export type PricingUnit = (typeof PricingUnit)[keyof typeof PricingUnit];
-export const TokenType = {
-    PROMPT: "PROMPT",
-    COMPLETION: "COMPLETION",
-    TOTAL: "TOTAL"
-} as const;
-export type TokenType = (typeof TokenType)[keyof typeof TokenType];
+export type ScoreDataType = (typeof ScoreDataType)[keyof typeof ScoreDataType];
 export const DatasetStatus = {
     ACTIVE: "ACTIVE",
     ARCHIVED: "ARCHIVED"
@@ -96,7 +91,7 @@ export type AuditLog = {
     updated_at: Generated<Timestamp>;
     user_id: string;
     project_id: string;
-    user_project_role: MembershipRole;
+    user_project_role: ProjectRole;
     resource_type: string;
     resource_id: string;
     action: string;
@@ -208,17 +203,10 @@ export type LlmApiKeys = {
     secret_key: string;
     project_id: string;
 };
-export type Membership = {
-    project_id: string;
-    user_id: string;
-    role: MembershipRole;
-    created_at: Generated<Timestamp>;
-    updated_at: Generated<Timestamp>;
-};
 export type MembershipInvitation = {
     id: string;
     email: string;
-    role: MembershipRole;
+    role: ProjectRole;
     project_id: string;
     sender_id: string | null;
     created_at: Generated<Timestamp>;
@@ -253,6 +241,7 @@ export type Observation = {
     status_message: string | null;
     version: string | null;
     created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
     model: string | null;
     internal_model: string | null;
     modelParameters: unknown | null;
@@ -300,6 +289,7 @@ export type ObservationView = {
     calculated_output_cost: string | null;
     calculated_total_cost: string | null;
     latency: number | null;
+    time_to_first_token: number | null;
 };
 export type PosthogIntegration = {
     project_id: string;
@@ -309,14 +299,6 @@ export type PosthogIntegration = {
     enabled: boolean;
     created_at: Generated<Timestamp>;
 };
-export type Pricing = {
-    id: string;
-    model_name: string;
-    pricing_unit: Generated<PricingUnit>;
-    price: string;
-    currency: Generated<string>;
-    token_type: TokenType;
-};
 export type Project = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -324,6 +306,13 @@ export type Project = {
     name: string;
     cloud_config: unknown | null;
 };
+export type ProjectMembership = {
+    project_id: string;
+    user_id: string;
+    role: ProjectRole;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Prompt = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -342,12 +331,32 @@ export type Prompt = {
 export type Score = {
     id: string;
     timestamp: Generated<Timestamp>;
+    project_id: string;
     name: string;
     value: number;
     source: ScoreSource;
+    author_user_id: string | null;
     comment: string | null;
     trace_id: string;
     observation_id: string | null;
+    config_id: string | null;
+    string_value: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    data_type: Generated<ScoreDataType>;
+};
+export type ScoreConfig = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    data_type: ScoreDataType;
+    is_archived: Generated<boolean>;
+    min_value: number | null;
+    max_value: number | null;
+    categories: unknown | null;
+    description: string | null;
 };
 export type Session = {
     id: string;
@@ -378,6 +387,8 @@ export type Trace = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
 };
 export type TraceSession = {
     id: string;
@@ -437,14 +448,14 @@ export type DB = {
     job_executions: JobExecution;
     llm_api_keys: LlmApiKeys;
     membership_invitations: MembershipInvitation;
-    memberships: Membership;
     models: Model;
     observations: Observation;
     observations_view: ObservationView;
     posthog_integrations: PosthogIntegration;
-    pricings: Pricing;
+    project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
+    score_configs: ScoreConfig;
     scores: Score;
     Session: Session;
     sso_configs: SsoConfig;

packages/shared/prisma/migrations/20240503125742_traces_add_createdat_updatedat/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "traces" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240503130335_traces_index_created_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_created_at_idx" ON "traces"("created_at");

packages/shared/prisma/migrations/20240503130520_traces_index_updated_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_updated_at_idx" ON "traces"("updated_at");

packages/shared/prisma/migrations/20240508132621_scores_add_project_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "project_id" TEXT;

packages/shared/prisma/migrations/20240508132735_scores_add_projectid_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_project_id_idx" ON "scores"("project_id");

packages/shared/prisma/migrations/20240508132736_scores_backfill_project_id/migration.sql

@@ -0,0 +1,5 @@
+-- Backfill project_id on existing scores based on linked trace_id
+UPDATE scores
+SET project_id = traces.project_id
+FROM traces 
+WHERE scores.trace_id = traces.id AND scores.project_id IS NULL;

packages/shared/prisma/migrations/20240512151529_rename_memberships_to_project_memberships/migration.sql

@@ -0,0 +1,15 @@
+-- Rename table, will cause a downtime as it needs to be locked and synced with TRPC API
+ALTER TABLE "memberships"
+RENAME TO "project_memberships";
+
+-- AlterTable
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_pkey" TO "project_memberships_pkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_project_id_fkey" TO "project_memberships_project_id_fkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_user_id_fkey" TO "project_memberships_user_id_fkey";
+
+-- RenameIndex
+ALTER INDEX "memberships_user_id_idx" RENAME TO "project_memberships_user_id_idx";

packages/shared/prisma/migrations/20240512155020_rename_enum_membership_role_to_project_role/migration.sql

@@ -0,0 +1,37 @@
+-- Goal: Rename the enum `MembershipRole` to `ProjectRole` and update all tables that use it
+
+-- Create a new Enum
+CREATE TYPE "ProjectRole" AS ENUM ('OWNER', 'ADMIN', 'MEMBER', 'VIEWER');
+
+-- Add a temporary column to tables (for each table)
+ALTER TABLE "audit_logs" ADD COLUMN "user_project_role_temp" text;
+ALTER TABLE "membership_invitations" ADD COLUMN "role_temp" text;
+ALTER TABLE "project_memberships" ADD COLUMN "role_temp" text;
+
+-- Copy data over to temporary columns (for each table)
+UPDATE "audit_logs" SET "user_project_role_temp" = "user_project_role"::text;
+UPDATE "membership_invitations" SET "role_temp" = "role"::text;
+UPDATE "project_memberships" SET "role_temp" = "role"::text;
+
+-- Drop old columns
+ALTER TABLE "audit_logs" DROP COLUMN "user_project_role";
+ALTER TABLE "membership_invitations" DROP COLUMN "role";
+ALTER TABLE "project_memberships" DROP COLUMN "role";
+
+-- Rename temporary columns to old column names
+ALTER TABLE "audit_logs" RENAME COLUMN "user_project_role_temp" TO "user_project_role";
+ALTER TABLE "membership_invitations" RENAME COLUMN "role_temp" TO "role";
+ALTER TABLE "project_memberships" RENAME COLUMN "role_temp" TO "role";
+
+-- Convert the text columns to enum columns (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" TYPE "ProjectRole" USING "user_project_role"::"ProjectRole";
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+ALTER TABLE "project_memberships" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+
+-- Make the columns NOT NULL (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" SET NOT NULL;
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" SET NOT NULL;
+ALTER TABLE "project_memberships" ALTER COLUMN "role" SET NOT NULL;
+
+-- Now finally, you can drop your old enum
+DROP TYPE "MembershipRole";

packages/shared/prisma/migrations/20240512155021_add_pricing_gpt4o/migration.sql

@@ -0,0 +1,20 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o
+  ('b9854a5c92dc496b997d99d20', NULL, 'gpt-4o', '(?i)^(gpt-4o)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }'),
+  
+  -- gpt-4o-2024-05-13
+  ('b9854a5c92dc496b997d99d21', NULL, 'gpt-4o-2024-05-13', '(?i)^(gpt-4o-2024-05-13)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o-2024-05-13" }')
+  

packages/shared/prisma/migrations/20240512155021_scores_drop_fk_on_traces_and_observations/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_observation_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_trace_id_fkey";

packages/shared/prisma/migrations/20240512155022_scores_non_null_and_add_fk_project_id/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - Made the column `project_id` on table `scores` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "project_id" SET NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240513082203_scores_unique_id_and_projectid_instead_of_id_and_traceid/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- DropIndex
+DROP INDEX "scores_id_trace_id_key";

packages/shared/prisma/migrations/20240513082204_scores_unique_id_and_projectid_instead_of_id_and_traceid_index/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- CreateIndex
+CREATE UNIQUE INDEX CONCURRENTLY "scores_id_project_id_key" ON "scores"("id", "project_id");

packages/shared/prisma/migrations/20240513082205_observations_view_add_time_to_first_token/migration.sql

@@ -0,0 +1,61 @@
+CREATE OR REPLACE VIEW "observations_view" AS
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency",
+    CASE WHEN o.completion_start_time IS NOT NULL AND o.start_time IS NOT NULL THEN EXTRACT(EPOCH FROM (completion_start_time - start_time))::double precision ELSE NULL END as "time_to_first_token"
+    
+FROM
+    observations o
+LEFT JOIN LATERAL (
+    SELECT
+        models.*
+    FROM
+        models
+    WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
+    AND models.model_name = o.internal_model
+    AND (models.start_date < o.start_time OR models.start_date IS NULL)
+    AND o.unit::TEXT = models.unit
+    ORDER BY
+        models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+) m ON TRUE
+
+
+-- requirements:
+-- 1. The view should return all columns from the observations table
+-- 2. The view should match with only one model for each observation if:
+--     a. The model has the same project_id as the observation, otherwise the model without project_id. 
+--     b. The model has the same model_name as the observation
+--     c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
+--     d. The model has the same unit as the observation

packages/shared/prisma/migrations/20240522081254_scores_add_author_user_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "author_user_id" TEXT;

packages/shared/prisma/migrations/20240522095738_scores_add_author_user_id_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_author_user_id_idx" ON "scores"("author_user_id");

packages/shared/prisma/migrations/20240523142425_score_config_add_table/migration.sql

@@ -0,0 +1,42 @@
+-- CreateEnum
+CREATE TYPE "ScoreDataType" AS ENUM ('CATEGORICAL', 'NUMERIC');
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "config_id" TEXT,
+ADD COLUMN     "data_type" "ScoreDataType" NOT NULL DEFAULT 'NUMERIC',
+ADD COLUMN     "string_value" TEXT;
+
+-- CreateTable
+CREATE TABLE "score_configs" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "data_type" "ScoreDataType" NOT NULL,
+    "is_archived" BOOLEAN NOT NULL DEFAULT false,
+    "min_value" DOUBLE PRECISION,
+    "max_value" DOUBLE PRECISION,
+    "categories" JSONB,
+    "description" TEXT,
+
+    CONSTRAINT "score_configs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "score_configs_data_type_idx" ON "score_configs"("data_type");
+
+-- CreateIndex
+CREATE INDEX "score_configs_is_archived_idx" ON "score_configs"("is_archived");
+
+-- CreateIndex
+CREATE INDEX "score_configs_project_id_idx" ON "score_configs"("project_id");
+
+-- CreateIndex
+CREATE INDEX "score_configs_categories_idx" ON "score_configs"("categories");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "score_configs_id_project_id_key" ON "score_configs"("id", "project_id");
+
+-- AddForeignKey
+ALTER TABLE "score_configs" ADD CONSTRAINT "score_configs_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240523142524_scores_add_config_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_config_id_idx" ON "scores"("config_id");

packages/shared/prisma/migrations/20240523142610_scores_add_fk_scores_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_config_id_fkey" FOREIGN KEY ("config_id") REFERENCES "score_configs"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240524154058_scores_source_enum_add_annotation/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'ANNOTATION';

packages/shared/prisma/migrations/20240524156058_scores_source_backfill_annotation_for_review/migration.sql

@@ -0,0 +1,4 @@
+-- Backfill the scores source for 'REVIEW' to be 'ANNOTATION'
+UPDATE "scores"
+SET "source" = 'ANNOTATION'::"ScoreSource"
+WHERE "source" = 'REVIEW'::"ScoreSource";

packages/shared/prisma/migrations/20240524165931_scores_source_enum_drop_review/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - The values [REVIEW] on the enum `ScoreSource` will be removed. If these variants are still used in the database, this will fail.
+
+*/
+-- AlterEnum
+BEGIN;
+CREATE TYPE "ScoreSource_new" AS ENUM ('ANNOTATION', 'API', 'EVAL');
+ALTER TABLE "scores" ALTER COLUMN "source" TYPE "ScoreSource_new" USING ("source"::text::"ScoreSource_new");
+ALTER TYPE "ScoreSource" RENAME TO "ScoreSource_old";
+ALTER TYPE "ScoreSource_new" RENAME TO "ScoreSource";
+DROP TYPE "ScoreSource_old";
+COMMIT;

packages/shared/prisma/migrations/20240524190433_job_executions_add_fk_index_config_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_configuration_id_idx" ON "job_executions"("job_configuration_id");

packages/shared/prisma/migrations/20240524190434_job_executions_add_fk_index_score_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_output_score_id_idx" ON "job_executions"("job_output_score_id");

packages/shared/prisma/migrations/20240524190435_job_executions_add_fk_index_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_job_input_trace_id_idx" ON "job_executions"("job_input_trace_id");

packages/shared/prisma/migrations/20240524190436_job_executions_index_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_created_at_idx" ON "job_executions"("created_at");

packages/shared/prisma/migrations/20240528214726_add_cursor_new_columns_observations/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240528214727_add_cursor_new_columns_scores/migration.sql

@@ -0,0 +1,4 @@
+
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240528214728_add_cursor_index_01/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "audit_logs_updated_at_idx" ON "audit_logs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_02/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_created_at_idx" ON "dataset_items"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_03/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_updated_at_idx" ON "dataset_items"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_04/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_created_at_idx" ON "dataset_run_items"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_05/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_updated_at_idx" ON "dataset_run_items"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_06/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_created_at_idx" ON "dataset_runs"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_07/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_runs_updated_at_idx" ON "dataset_runs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_08/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_created_at_idx" ON "datasets"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_09/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "datasets_updated_at_idx" ON "datasets"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_10/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "job_executions_updated_at_idx" ON "job_executions"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_11/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_updated_at_idx" ON "observations"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_12/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_created_at_idx" ON "prompts"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_13/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_updated_at_idx" ON "prompts"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_14/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_created_at_idx" ON "score_configs"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_15/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "score_configs_updated_at_idx" ON "score_configs"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_16/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_created_at_idx" ON "scores"("created_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_17/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_updated_at_idx" ON "scores"("updated_at");

packages/shared/prisma/migrations/20240528214728_add_cursor_index_18/migration.sql

@@ -0,0 +1,3 @@
+
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "trace_sessions_updated_at_idx" ON "trace_sessions"("updated_at");

packages/shared/prisma/migrations/20240530212419_model_price_anthropic_via_google_vertex/migration.sql

@@ -0,0 +1,7 @@
+-- Google Vertex uses @ to separate model name and version
+
+UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-haiku(-|@)?20240307)$' WHERE "id" = 'cltr0w45b000008k1407o9qv1';
+
+UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-opus(-|@)?20240229)$' WHERE "id" = 'cltgy0iuw000008le3vod1hhy';
+
+UPDATE "public"."models" SET "match_pattern" = '(?i)^(claude-3-sonnet(-|@)?20240229)$' WHERE "id" = 'cltgy0pp6000108le56se7bl3';

packages/shared/prisma/migrations/20240603212024_dataset_items_add_index_source_trace_id/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_items_source_trace_id_idx" ON "dataset_items" USING HASH ("source_trace_id");

packages/shared/prisma/migrations/20240604133338_scores_add_index_name/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "scores_project_id_name_idx" ON "scores"("project_id", "name");

packages/shared/prisma/migrations/20240604133339_score_data_type_add_boolean/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "ScoreDataType" ADD VALUE 'BOOLEAN';

packages/shared/prisma/migrations/20240604133340_backfill_manual_scores/migration.sql

@@ -0,0 +1,50 @@
+-- Create manual-score config for any project with manual scores and link config_id to scores
+BEGIN;
+WITH project_configs AS (
+INSERT INTO score_configs (id,
+		project_id,
+		name,
+		data_type,
+		is_archived,
+		min_value,
+		max_value,
+		description)
+SELECT
+	gen_random_uuid () AS id,
+	s.project_id,
+	'manual-score',
+	'NUMERIC',
+	FALSE,
+	- 1,
+	1,
+	'Langfuse legacy annotation score.'
+FROM ( SELECT DISTINCT
+		project_id
+	FROM
+		scores
+	WHERE
+		name = 'manual-score'
+		AND source = 'ANNOTATION') s
+WHERE
+	NOT EXISTS (
+		SELECT
+			1
+		FROM
+			score_configs sc
+		WHERE
+			sc.name = 'manual-score'
+			AND sc.project_id = s.project_id)
+	RETURNING
+		id,
+		project_id
+)
+UPDATE
+	scores
+SET
+	config_id = pc.id
+FROM
+	project_configs pc
+WHERE
+	scores.project_id = pc.project_id
+	AND scores.name = 'manual-score';
+COMMIT;

packages/shared/prisma/migrations/20240606090858_pricings_add_latest_gemini_models/migration.sql

@@ -0,0 +1,5 @@
+-- does not include pricing yet, will be added as soon as it is calculated at ingestion time
+
+INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30djsn0000w9mzebiv41we', 'gemini-1.5-flash', '(?i)^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');
+
+INSERT INTO "public"."models" ("id", "model_name", "match_pattern", "unit") VALUES ('clx30hkrx0000w9mz7lqi0ial', 'gemini-1.5-pro', '(?i)^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS');

packages/shared/prisma/migrations/20240606093356_drop_unused_pricings_table/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - You are about to drop the `pricings` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropTable
+DROP TABLE "pricings";
+
+-- DropEnum
+DROP TYPE "PricingUnit";
+
+-- DropEnum
+DROP TYPE "TokenType";

packages/shared/prisma/migrations/20240606133011_remove_trace_fkey_datasetrunitems/migration.sql

@@ -0,0 +1,3 @@
+-- DropForeignKey
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_trace_id_fkey";
+ALTER TABLE "dataset_run_items" DROP CONSTRAINT "dataset_run_items_observation_id_fkey";

packages/shared/prisma/schema.prisma

@@ -3,7 +3,7 @@
 
 generator client {
     provider        = "prisma-client-js"
-    previewFeatures = ["tracing", "views"]
+    previewFeatures = ["tracing", "views", "relationJoins"]
 }
 
 datasource db {
@@ -63,21 +63,21 @@ model Session {
 }
 
 model User {
-    id            String                 @id @default(cuid())
-    name          String?
-    email         String?                @unique
-    emailVerified DateTime?              @map("email_verified")
-    password      String?
-    image         String?
-    admin         Boolean                @default(false)
-    accounts      Account[]
-    sessions      Session[]
-    memberships   Membership[]
-    invitations   MembershipInvitation[]
-    createdAt     DateTime               @default(now()) @map("created_at")
-    updatedAt     DateTime               @default(now()) @updatedAt @map("updated_at")
-    featureFlags  String[]               @default([]) @map("feature_flags")
-    AuditLog      AuditLog[]
+    id                 String                 @id @default(cuid())
+    name               String?
+    email              String?                @unique
+    emailVerified      DateTime?              @map("email_verified")
+    password           String?
+    image              String?
+    admin              Boolean                @default(false)
+    accounts           Account[]
+    sessions           Session[]
+    projectMemberships ProjectMembership[]
+    invitations        MembershipInvitation[]
+    createdAt          DateTime               @default(now()) @map("created_at")
+    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
+    featureFlags       String[]               @default([]) @map("feature_flags")
+    AuditLog           AuditLog[]
 
     @@map("users")
 }
@@ -97,7 +97,7 @@ model Project {
     updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
     name               String
     cloudConfig        Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members            Membership[]
+    projectMembers     ProjectMembership[]
     traces             Trace[]
     observations       Observation[]
     apiKeys            ApiKey[]
@@ -113,6 +113,8 @@ model Project {
     JobExecution       JobExecution[]
     LlmApiKeys         LlmApiKeys[]
     PosthogIntegration PosthogIntegration[]
+    Score              Score[]
+    scoreConfig        ScoreConfig[]
 
     @@map("projects")
 }
@@ -154,37 +156,37 @@ model LlmApiKeys {
     @@map("llm_api_keys")
 }
 
-model Membership {
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userId    String         @map("user_id")
-    user      User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    role      MembershipRole
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+model ProjectMembership {
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId    String      @map("user_id")
+    user      User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role      ProjectRole
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@id([projectId, userId])
     @@index([userId])
-    @@map("memberships")
+    @@map("project_memberships")
 }
 
 model MembershipInvitation {
-    id        String         @id @unique @default(cuid())
+    id        String      @id @unique @default(cuid())
     email     String
-    role      MembershipRole
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    senderId  String?        @map("sender_id")
-    sender    User?          @relation(fields: [senderId], references: [id], onDelete: SetNull)
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+    role      ProjectRole
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    senderId  String?     @map("sender_id")
+    sender    User?       @relation(fields: [senderId], references: [id], onDelete: SetNull)
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@index([projectId])
     @@index([email])
     @@map("membership_invitations")
 }
 
-enum MembershipRole {
+enum ProjectRole {
     OWNER
     ADMIN
     MEMBER
@@ -204,6 +206,7 @@ model TraceSession {
     @@id([id, projectId])
     @@index([projectId])
     @@index([createdAt])
+    @@index([updatedAt])
     @@map("trace_sessions")
 }
 
@@ -227,11 +230,11 @@ model Trace {
     output     Json?
     sessionId  String?       @map("session_id")
     session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
+    createdAt  DateTime      @default(now()) @map("created_at")
+    updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    scores          Score[]
-    DatasetRunItems DatasetRunItems[]
-    DatasetItem     DatasetItem[]
-    JobExecution    JobExecution[]
+    DatasetItem  DatasetItem[]
+    JobExecution JobExecution[]
 
     @@index([projectId])
     @@index([sessionId])
@@ -241,6 +244,8 @@ model Trace {
     @@index([externalId])
     @@index(timestamp)
     @@index(release)
+    @@index(updatedAt)
+    @@index(createdAt)
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("traces")
 }
@@ -272,6 +277,8 @@ view TraceView {
 }
 
 // Update ObservationView below when making changes to this model!
+// traceId is optional only due to timing during data injestion 
+// (traceId is not necessarily known at the time of observation creation)
 model Observation {
     id                  String           @id @default(cuid())
     traceId             String?          @map("trace_id")
@@ -286,25 +293,24 @@ model Observation {
     statusMessage       String?          @map("status_message")
     version             String?
     createdAt           DateTime         @default(now()) @map("created_at")
+    updatedAt           DateTime         @default(now()) @updatedAt @map("updated_at")
 
     // GENERATION ONLY
     model               String?
-    internalModel       String?           @map("internal_model")
+    internalModel       String?       @map("internal_model")
     modelParameters     Json?
     input               Json?
     output              Json?
-    promptTokens        Int               @default(0) @map("prompt_tokens")
-    completionTokens    Int               @default(0) @map("completion_tokens")
-    totalTokens         Int               @default(0) @map("total_tokens")
+    promptTokens        Int           @default(0) @map("prompt_tokens")
+    completionTokens    Int           @default(0) @map("completion_tokens")
+    totalTokens         Int           @default(0) @map("total_tokens")
     unit                String?
-    inputCost           Decimal?          @map("input_cost")
-    outputCost          Decimal?          @map("output_cost")
-    totalCost           Decimal?          @map("total_cost")
-    completionStartTime DateTime?         @map("completion_start_time")
-    scores              Score[]
-    project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    inputCost           Decimal?      @map("input_cost")
+    outputCost          Decimal?      @map("output_cost")
+    totalCost           Decimal?      @map("total_cost")
+    completionStartTime DateTime?     @map("completion_start_time")
+    project             Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
     derivedDatasetItems DatasetItem[]
-    datasetRunItems     DatasetRunItems[]
 
     promptId String? @map("prompt_id")
     prompt   Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
@@ -318,6 +324,7 @@ model Observation {
     @@index([type])
     @@index(startTime)
     @@index(createdAt)
+    @@index(updatedAt)
     @@index(projectId)
     @@index(parentObservationId)
     @@index(model)
@@ -368,6 +375,7 @@ view ObservationView {
     calculatedOutputCost Decimal? @map("calculated_output_cost")
     calculatedTotalCost  Decimal? @map("calculated_total_cost")
     latency              Float?   @map("latency")
+    timeToFirstToken     Float?   @map("time_to_first_token")
 
     @@map("observations_view")
 }
@@ -388,52 +396,73 @@ enum ObservationLevel {
 model Score {
     id            String         @id @default(cuid())
     timestamp     DateTime       @default(now())
+    projectId     String         @map("project_id")
+    project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
     value         Float
     source        ScoreSource
+    authorUserId  String?        @map("author_user_id")
     comment       String?
     traceId       String         @map("trace_id")
-    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
     observationId String?        @map("observation_id")
-    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    configId      String?        @map("config_id")
+    stringValue   String?        @map("string_value")
+    createdAt     DateTime       @default(now()) @map("created_at")
+    updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
+    dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
     JobExecution  JobExecution[]
+    scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
 
-    @@unique([id, traceId]) // used for upsert
+    @@unique([id, projectId]) // used for upserts via prisma
     @@index(timestamp)
     @@index([value])
+    @@index([projectId])
+    @@index([projectId, name])
+    @@index([authorUserId])
+    @@index([configId])
     @@index([traceId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([source])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("scores")
 }
 
 enum ScoreSource {
+    ANNOTATION
     API
-    REVIEW
     EVAL
 }
 
-enum PricingUnit {
-    PER_1000_TOKENS
-    PER_1000_CHARS
+model ScoreConfig {
+    id          String        @id @default(cuid())
+    createdAt   DateTime      @default(now()) @map("created_at")
+    updatedAt   DateTime      @default(now()) @updatedAt @map("updated_at")
+    projectId   String        @map("project_id")
+    project     Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    name        String
+    dataType    ScoreDataType @map("data_type")
+    isArchived  Boolean       @default(false) @map("is_archived")
+    minValue    Float?        @map("min_value")
+    maxValue    Float?        @map("max_value")
+    categories  Json?         @map("categories")
+    description String?
+    score       Score[]
+
+    @@unique([id, projectId]) // used for upserts via prisma
+    @@index([dataType])
+    @@index([isArchived])
+    @@index([projectId])
+    @@index([categories])
+    @@index([createdAt])
+    @@index([updatedAt])
+    @@map("score_configs")
 }
 
-enum TokenType {
-    PROMPT
-    COMPLETION
-    TOTAL
-}
-
-model Pricing {
-    id          String      @id @default(cuid())
-    modelName   String      @map("model_name")
-    pricingUnit PricingUnit @default(PER_1000_TOKENS) @map("pricing_unit")
-    price       Decimal
-    currency    String      @default("USD")
-    tokenType   TokenType   @map("token_type")
-
-    @@index(modelName)
-    @@map("pricings")
+enum ScoreDataType {
+    CATEGORICAL
+    NUMERIC
+    BOOLEAN
 }
 
 model CronJobs {
@@ -459,6 +488,8 @@ model Dataset {
 
     @@unique([projectId, name])
     @@index([projectId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("datasets")
 }
 
@@ -478,8 +509,11 @@ model DatasetItem {
     updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
     datasetRunItems     DatasetRunItems[]
 
+    @@index([sourceTraceId], type: Hash)
     @@index([sourceObservationId], type: Hash)
     @@index([datasetId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("dataset_items")
 }
 
@@ -501,26 +535,28 @@ model DatasetRuns {
 
     @@unique([datasetId, name])
     @@index([datasetId], type: Hash)
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("dataset_runs")
 }
 
 model DatasetRunItems {
-    id            String       @id @default(cuid())
-    datasetRunId  String       @map("dataset_run_id")
-    datasetRun    DatasetRuns  @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
-    datasetItemId String       @map("dataset_item_id")
-    datasetItem   DatasetItem  @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: Cascade)
-    createdAt     DateTime     @default(now()) @map("created_at")
-    updatedAt     DateTime     @default(now()) @updatedAt @map("updated_at")
+    id            String      @id @default(cuid())
+    datasetRunId  String      @map("dataset_run_id")
+    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetItemId String      @map("dataset_item_id")
+    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    traceId       String      @map("trace_id")
+    observationId String?     @map("observation_id")
+    createdAt     DateTime    @default(now()) @map("created_at")
+    updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@index([datasetRunId], type: Hash)
     @@index([datasetItemId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([traceId])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("dataset_run_items")
 }
 
@@ -563,6 +599,8 @@ model Prompt {
     @@index([projectId, name, version])
     @@index([projectId, id])
     @@index([projectId])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("prompts")
 }
@@ -594,22 +632,23 @@ model Model {
 }
 
 model AuditLog {
-    id              String         @id @default(cuid())
-    createdAt       DateTime       @default(now()) @map("created_at")
-    updatedAt       DateTime       @default(now()) @updatedAt @map("updated_at")
-    userId          String         @map("user_id")
-    user            User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    projectId       String         @map("project_id")
-    project         Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userProjectRole MembershipRole @map("user_project_role")
-    resourceType    String         @map("resource_type")
-    resourceId      String         @map("resource_id")
+    id              String      @id @default(cuid())
+    createdAt       DateTime    @default(now()) @map("created_at")
+    updatedAt       DateTime    @default(now()) @updatedAt @map("updated_at")
+    userId          String      @map("user_id")
+    user            User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    projectId       String      @map("project_id")
+    project         Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userProjectRole ProjectRole @map("user_project_role")
+    resourceType    String      @map("resource_type")
+    resourceId      String      @map("resource_id")
     action          String
     before          String? //stringified JSON
     after           String? // stringified JSON
 
     @@index([projectId])
     @@index([createdAt])
+    @@index([updatedAt])
     @@map("audit_logs")
 }
 
@@ -703,6 +742,11 @@ model JobExecution {
     @@index([projectId, status])
     @@index([projectId, id])
     @@index([projectId])
+    @@index([jobConfigurationId])
+    @@index([jobOutputScoreId])
+    @@index([jobInputTraceId])
+    @@index([createdAt])
+    @@index([updatedAt])
     @@map("job_executions")
 }
 

packages/shared/prisma/seed.ts

@@ -4,6 +4,7 @@ import {
   type Prisma,
   ObservationType,
   ScoreSource,
+  ScoreDataType,
 } from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
@@ -12,6 +13,7 @@ import { chunk } from "lodash";
 import { v4 } from "uuid";
 import { ModelUsageUnit } from "../src";
 import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
+import { encrypt } from "../src/encryption";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -38,6 +40,7 @@ async function main() {
       name: "Demo User",
       email: "demo@langfuse.com",
       password: await hash("password", 12),
+      image: "https://static.langfuse.com/langfuse-dev%2Fexample-avatar.png",
     },
   });
 
@@ -51,7 +54,7 @@ async function main() {
     create: {
       id: seedProjectId,
       name: "llm-app",
-      members: {
+      projectMembers: {
         create: {
           role: "OWNER",
           userId: user.id,
@@ -72,7 +75,7 @@ async function main() {
       name: "summary-prompt",
       project: { connect: { id: seedProjectId } },
       prompt: "prompt {{variable}} {{anotherVariable}}",
-      labels: ["production"],
+      labels: ["production", "latest"],
       version: 1,
       createdBy: "user-1",
     },
@@ -110,7 +113,7 @@ async function main() {
       create: {
         id: "239ad00f-562f-411d-af14-831c75ddd875",
         name: "demo-app",
-        members: {
+        projectMembers: {
           create: {
             role: "OWNER",
             userId: user.id,
@@ -143,6 +146,11 @@ async function main() {
       });
     }
 
+    const configIdsAndNames = await generateConfigsForProject([
+      project1,
+      project2,
+    ]);
+
     const promptIds = await generatePromptsForProject([project1, project2]);
 
     const envTags = [null, "development", "staging", "production"];
@@ -156,7 +164,8 @@ async function main() {
       colorTags,
       project1,
       project2,
-      promptIds
+      promptIds,
+      configIdsAndNames
     );
 
     console.log(
@@ -165,6 +174,24 @@ async function main() {
 
     await uploadObjects(traces, observations, scores, sessions, events);
 
+    // If openai key is in environment, add it to the projects LLM API keys
+    const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+
+    if (OPENAI_API_KEY) {
+      await prisma.llmApiKeys.create({
+        data: {
+          projectId: project1.id,
+          secretKey: encrypt(OPENAI_API_KEY),
+          displaySecretKey: getDisplaySecretKey(OPENAI_API_KEY),
+          provider: "openai",
+        },
+      });
+    } else {
+      console.warn(
+        "No OPENAI_API_KEY found in environment. Skipping seeding LLM API key."
+      );
+    }
+
     // add eval objects
     const evalTemplate = await prisma.evalTemplate.upsert({
       where: {
@@ -434,13 +461,15 @@ function createObjects(
   colorTags: (string | null)[],
   project1: Project,
   project2: Project,
-  promptIds: Map<string, string[]>
+  promptIds: Map<string, string[]>,
+  configIdsAndNames: Map<string, { name: string; id: string }[]>
 ) {
   const traces: Prisma.TraceCreateManyInput[] = [];
   const observations: Prisma.ObservationCreateManyInput[] = [];
   const scores: Prisma.ScoreCreateManyInput[] = [];
   const sessions: Prisma.TraceSessionCreateManyInput[] = [];
   const events: Prisma.ObservationCreateManyInput[] = [];
+  const configs: Prisma.ScoreConfigCreateManyInput[] = [];
 
   for (let i = 0; i < traceVolume; i++) {
     // print progress to console with a progress bar that refreshes every 10 iterations
@@ -459,7 +488,7 @@ function createObjects(
     const session =
       Math.random() > 0.3
         ? {
-            id: `session-${i % 10}`,
+            id: `session-${i % 3}`,
             projectId: projectId,
           }
         : undefined;
@@ -492,15 +521,28 @@ function createObjects(
 
     traces.push(trace);
 
+    const configArray = configIdsAndNames.get(projectId) ?? [];
+    const randomIndex = Math.floor(Math.random() * 3);
+    const config =
+      configArray.length >= randomIndex - 1 && configArray[randomIndex];
+    const { name: annotationScoreName, id: configId } = config || {
+      name: "manual-score",
+      id: undefined,
+    };
+
     const traceScores = [
       ...(Math.random() > 0.5
         ? [
             {
               traceId: trace.id,
-              name: "manual-score",
+              name: annotationScoreName,
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
-              source: ScoreSource.REVIEW,
+              source: ScoreSource.ANNOTATION,
+              projectId,
+              authorUserId: `user-${i}`,
+              dataType: ScoreDataType.NUMERIC,
+              ...(configId ? { configId } : {}),
             },
           ]
         : []),
@@ -512,6 +554,8 @@ function createObjects(
               value: Math.floor(Math.random() * 10) - 5,
               timestamp: traceTs,
               source: ScoreSource.API,
+              projectId,
+              dataType: ScoreDataType.NUMERIC,
             },
           ]
         : []),
@@ -526,9 +570,9 @@ function createObjects(
       const spanTsStart = new Date(
         traceTs.getTime() + Math.floor(Math.random() * 30)
       );
-      // random duration of upto 30ms
+      // random duration of upto 5000ms
       const spanTsEnd = new Date(
-        spanTsStart.getTime() + Math.floor(Math.random() * 30)
+        spanTsStart.getTime() + Math.floor(Math.random() * 5000)
       );
 
       const span = {
@@ -572,6 +616,13 @@ function createObjects(
                 (spanTsEnd.getTime() - generationTsStart.getTime())
             )
         );
+        // somewhere in the middle
+        const generationTsCompletionStart = new Date(
+          generationTsStart.getTime() +
+            Math.floor(
+              (generationTsEnd.getTime() - generationTsStart.getTime()) / 3
+            )
+        );
 
         const promptTokens = Math.floor(Math.random() * 1000) + 300;
         const completionTokens = Math.floor(Math.random() * 500) + 100;
@@ -590,7 +641,9 @@ function createObjects(
         const model = models[Math.floor(Math.random() * models.length)];
         const promptId =
           promptIds.get(projectId)![
-            Math.floor(Math.random() * promptIds.get(projectId)!.length)
+            Math.floor(
+              Math.random() * Math.floor(promptIds.get(projectId)!.length / 2)
+            )
           ];
 
         const generation = {
@@ -598,6 +651,8 @@ function createObjects(
           id: `generation-${v4()}`,
           startTime: generationTsStart,
           endTime: generationTsEnd,
+          completionStartTime:
+            Math.random() > 0.5 ? generationTsCompletionStart : undefined,
           name: `generation-${i}-${j}-${k}`,
           projectId: trace.projectId,
           promptId: promptId,
@@ -693,6 +748,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
         if (Math.random() > 0.6)
           scores.push({
@@ -701,6 +757,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
 
         for (let l = 0; l < Math.floor(Math.random() * 2); l++) {
@@ -737,6 +794,7 @@ function createObjects(
     traces,
     observations,
     scores,
+    configs,
     sessions: uniqueSessions,
     events,
   };
@@ -764,7 +822,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 1 content",
       name: "Prompt 1",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -773,7 +831,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 2 content",
       name: "Prompt 2",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -782,7 +840,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 3 content",
       name: "Prompt 3 by API",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -791,7 +849,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 4 content",
       name: "Prompt 4",
       version: 1,
-      labels: ["production"],
+      labels: ["production", "latest"],
       tags: ["tag1", "tag2"],
     },
   ];
@@ -859,6 +917,7 @@ async function generatePrompts(project: Project) {
         frequencyPenalty: 0.5,
       },
       version: 3,
+      labels: ["production", "latest"],
     },
   ];
 
@@ -908,7 +967,7 @@ async function generatePrompts(project: Project) {
         prompt: `${promptName} version ${i} content`,
         name: promptName,
         version: i,
-        labels: i === 20 ? ["production"] : [],
+        labels: i === 20 ? ["production", "latest"] : [],
       },
       update: {
         id: promptId,
@@ -918,3 +977,80 @@ async function generatePrompts(project: Project) {
   }
   return promptIds;
 }
+
+async function generateConfigsForProject(projects: Project[]) {
+  const projectIdsToConfigs: Map<string, { name: string; id: string }[]> =
+    new Map();
+
+  await Promise.all(
+    projects.map(async (project) => {
+      const configNameAndId = await generateConfigs(project);
+      projectIdsToConfigs.set(project.id, configNameAndId);
+    })
+  );
+  return projectIdsToConfigs;
+}
+
+async function generateConfigs(project: Project) {
+  const configNameAndId: { name: string; id: string }[] = [];
+
+  const configs = [
+    {
+      id: `config-${v4()}`,
+      name: "manual-score",
+      dataType: ScoreDataType.NUMERIC,
+      projectId: project.id,
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Accuracy",
+      dataType: ScoreDataType.CATEGORICAL,
+      categories: [
+        { label: "Incorrect", value: 0 },
+        { label: "Partially Correct", value: 1 },
+        { label: "Correct", value: 2 },
+      ],
+      isArchived: false,
+    },
+    {
+      id: `config-${v4()}`,
+      projectId: project.id,
+      name: "Toxicity",
+      dataType: ScoreDataType.BOOLEAN,
+      categories: [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ],
+      description:
+        "Used to indicate if text was harmful or offensive in nature.",
+      isArchived: false,
+    },
+  ];
+
+  for (const config of configs) {
+    await prisma.scoreConfig.upsert({
+      where: {
+        id_projectId: {
+          projectId: config.projectId,
+          id: config.id,
+        },
+      },
+      create: {
+        id: config.id,
+        projectId: config.projectId,
+        name: config.name,
+        dataType: config.dataType,
+        categories: config.categories,
+        isArchived: config.isArchived,
+      },
+      update: {
+        id: config.id,
+      },
+    });
+    configNameAndId.push({ name: config.name, id: config.id });
+  }
+
+  return configNameAndId;
+}

packages/shared/src/features/evals/types.ts

@@ -1,5 +1,5 @@
 import z from "zod";
-import { ModelProvider } from "../..";
+import { ModelProvider, UIModelParams } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -77,26 +77,38 @@ export const availableEvalVariables = [
   },
 ];
 
-export const evalLLMModels = [
+export const evalLLMModels: UIModelParams[] = [
   {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-3.5-turbo",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-3.5-turbo", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
   },
   {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-4-turbo-preview",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4-turbo-preview", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
+  },
+  {
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4o", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
   },
 ] as const;
 
-export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
+export const EvalModelNames = z.enum([
+  "gpt-3.5-turbo",
+  "gpt-4-turbo-preview",
+  "gpt-4o",
+]);
 
 export const OutputSchema = z.object({
   reasoning: z.string(),

packages/shared/src/features/ingestion/types.ts

@@ -1,9 +1,9 @@
 import lodash from "lodash";
 import { z } from "zod";
 
-import { ModelUsageUnit } from "@langfuse/shared";
-import { NonEmptyString, jsonSchema } from "@/src/utils/zod";
-import { ObservationLevel } from "@langfuse/shared";
+import { NonEmptyString, jsonSchema } from "../../utils/zod";
+import { ModelUsageUnit } from "../../constants";
+import { ObservationLevel } from "@prisma/client";
 
 export const Usage = z.object({
   input: z.number().int().nullish(),
@@ -109,7 +109,7 @@ export const CreateGenerationBody = CreateSpanBody.extend({
       z.string(),
       z
         .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
-        .nullish(),
+        .nullish()
     )
     .nullish(),
   usage: usage,
@@ -131,7 +131,7 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
       z.string(),
       z
         .union([z.string(), z.number(), z.boolean(), z.array(z.string())])
-        .nullish(),
+        .nullish()
     )
     .nullish(),
   usage: usage,
@@ -195,7 +195,7 @@ export const LegacyGenerationsCreateSchema = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
     )
     .nullish(),
   prompt: jsonSchema.nullish(),
@@ -219,7 +219,7 @@ export const LegacyGenerationPatchSchema = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
     )
     .nullish(),
   prompt: jsonSchema.nullish(),
@@ -243,7 +243,7 @@ export const LegacyObservationBody = z.object({
   modelParameters: z
     .record(
       z.string(),
-      z.union([z.string(), z.number(), z.boolean()]).nullish(),
+      z.union([z.string(), z.number(), z.boolean()]).nullish()
     )
     .nullish(),
   input: jsonSchema.nullish(),

packages/shared/src/filterToPrisma.ts

@@ -1,7 +1,7 @@
 import { Prisma } from "@prisma/client";
 import {
   ColumnDefinition,
-  type TableNames as TableName,
+  type TableNames,
 } from "./interfaces/tableDefinition";
 import { FilterState } from "./types";
 import { filterOperators } from "./interfaces/filters";
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
   const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
   if (sql === Prisma.empty) {
@@ -43,7 +43,7 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName
+  table: TableNames
 ): Prisma.Sql {
   const internalFilters = filters.map((filter) => {
     // Get column definition to map column to internal name, e.g. "t.id"
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(
 
 const castValueToPostgresTypes = (
   column: ColumnDefinition,
-  table: TableName
+  table: TableNames
 ) => {
   return column.name === "type" &&
     (table === "observations" ||

packages/shared/src/index.ts

@@ -9,6 +9,9 @@ export * from "./filterToPrisma";
 export * from "./tracesTable";
 export * from "./server/auth";
 export * from "./observationsTable";
+export * from "./features/ingestion/types";
+export * from "./utils/zod";
+export * from "./utils/json";
 export { env } from "./env";
 
 // llm api

packages/shared/src/interfaces/tableDefinition.ts

@@ -42,6 +42,7 @@ export const tableNames = [
   "traces_parent_observation_scores",
   "sessions",
   "prompts",
+  "users",
 ] as const;
 
 export type TableNames = (typeof tableNames)[number];

packages/shared/src/server/llm/types.ts

@@ -20,19 +20,23 @@ export enum ChatMessageRole {
   Assistant = "assistant",
 }
 export type ModelParams = AnthropicModelParams | OpenAIModelParams;
-export type UIModelParams = Required<
-  AnthropicModelParams | OpenAIModelParams
-> & {
-  maxTemperature: number;
+
+type RecordWithEnabledFlag<T> = {
+  [K in keyof T]: { value: T[K]; enabled: boolean };
 };
+export type UIModelParams = RecordWithEnabledFlag<
+  Required<AnthropicModelParams | OpenAIModelParams> & {
+    maxTemperature: number;
+  }
+>;
 
 // Generic config
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
 
 export const ZodModelConfig = z.object({
-  max_tokens: z.number().optional(),
-  temperature: z.number().optional(),
-  top_p: z.number().optional(),
+  max_tokens: z.coerce.number().optional(),
+  temperature: z.coerce.number().optional(),
+  top_p: z.coerce.number().optional(),
 });
 
 // OpenAI
@@ -42,6 +46,8 @@ export type OpenAIModelParams = {
 } & ModelConfig;
 
 export const openAIModels = [
+  "gpt-4o",
+  "gpt-4o-2024-05-13",
   "gpt-4-turbo-preview",
   "gpt-4-1106-preview",
   "gpt-4-0613",

packages/shared/src/types.ts

@@ -30,4 +30,5 @@ export type TableName =
   | "sessions"
   | "scores"
   | "prompts"
-  | "dashboard";
+  | "dashboard"
+  | "users";

packages/shared/src/utils/json.ts

@@ -1,6 +1,6 @@
-import { type jsonSchema } from "@/src/utils/zod";
 import { type z } from "zod";
 import lodash from "lodash";
+import { jsonSchema } from "./zod";
 
 export const parseJson = (input: string) => {
   try {
@@ -37,7 +37,7 @@ export function deepParseJson(json: unknown): unknown {
         // Ensure we only iterate over the object's own properties
         if (Object.prototype.hasOwnProperty.call(json, key)) {
           (json as Record<string, unknown>)[key] = deepParseJson(
-            (json as Record<string, unknown>)[key],
+            (json as Record<string, unknown>)[key]
           );
         }
       }
@@ -50,7 +50,7 @@ export function deepParseJson(json: unknown): unknown {
 
 export const mergeJson = (
   json1?: z.infer<typeof jsonSchema>,
-  json2?: z.infer<typeof jsonSchema>,
+  json2?: z.infer<typeof jsonSchema>
 ) => {
   if (json1 === undefined) {
     return json2;

packages/shared/src/utils/zod.ts

@@ -26,7 +26,7 @@ const jsonSchemaNullable: z.ZodType<JsonNested> = z.lazy(() =>
     nestedLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
-  ]),
+  ])
 );
 
 // Root schema that does not allow nulls at the root level
@@ -35,20 +35,29 @@ export const jsonSchema: z.ZodType<Json> = z.lazy(() =>
     rootLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
-  ]),
+  ])
 );
 
 export const paginationZod = {
   page: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().default(1),
+    z.coerce.number().default(1)
   ),
   limit: z.preprocess(
     (x) => (x === "" ? undefined : x),
-    z.coerce.number().lte(100).default(50),
+    z.coerce.number().lte(100).default(50)
   ),
 };
 
+export const optionalPaginationZod = {
+  page: z
+    .preprocess((x) => (x === "" ? undefined : x), z.coerce.number())
+    .optional(),
+  limit: z
+    .preprocess((x) => (x === "" ? undefined : x), z.coerce.number())
+    .optional(),
+};
+
 export const noHtmlRegex = /<[^>]*>/;
 export const noHtmlCheck = (value: string) => !noHtmlRegex.test(value);
 

web/Dockerfile

@@ -5,7 +5,7 @@ FROM node:20-alpine AS alpine
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
 
 FROM alpine AS base
-RUN npm install turbo --global
+RUN npm install turbo@^1.13.3 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable

web/package.json

@@ -1,14 +1,13 @@
 {
   "name": "web",
-  "version": "2.37.2",
+  "version": "2.49.1",
   "private": true,
   "license": "MIT",
   "engines": {
     "node": ">=20"
   },
   "scripts": {
-    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml && cp generated/postman/collection.json public/postman-collection.json",
-    "build": "pnpm run copy-openapi-spec && INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
+    "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
     "lint": "dotenv -e ../.env -- next lint",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
@@ -23,17 +22,19 @@
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@aws-sdk/client-s3": "^3.550.0",
-    "@aws-sdk/lib-storage": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.568.0",
     "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@codemirror/lang-json": "^6.0.1",
-    "@headlessui/react": "^1.7.19",
+    "@codemirror/lint": "^6.8.0",
+    "@headlessui/react": "1.7.18",
+    "@headlessui/tailwindcss": "0.2.0",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.14",
-    "@langchain/core": "^0.1.58",
-    "@langchain/openai": "^0.0.28",
-    "@langfuse/shared": "workspace:*",
+    "@langchain/anthropic": "^0.1.21",
+    "@langchain/core": "^0.1.61",
+    "@langchain/openai": "^0.0.33",
     "@langfuse/ee": "workspace:*",
+    "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
     "@next-auth/prisma-adapter": "^1.0.7",
     "@radix-ui/react-accordion": "^1.1.2",
@@ -55,53 +56,58 @@
     "@radix-ui/react-switch": "^1.0.3",
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
+    "@radix-ui/react-toggle-group": "^1.0.4",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.16",
-    "@react-email/render": "^0.0.12",
+    "@react-email/components": "^0.0.18",
+    "@react-email/render": "^0.0.14",
+    "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.110.1",
-    "@sentry/node": "^7.110.1",
-    "@sentry/profiling-node": "^7.110.1",
-    "@sentry/types": "^7.110.1",
-    "@t3-oss/env-nextjs": "^0.8.0",
-    "@tailwindcss/forms": "^0.5.7",
+    "@sentry/nextjs": "^7.113.0",
+    "@sentry/node": "^7.113.0",
+    "@sentry/profiling-node": "^7.113.0",
+    "@sentry/types": "^7.113.0",
+    "@t3-oss/env-nextjs": "^0.10.1",
+    "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
-    "@tremor/react": "3.11.1",
+    "@tremor/react": "3.16.2",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
     "@uiw/codemirror-theme-github": "^4.21.25",
+    "@uiw/codemirror-theme-tokyo-night": "^4.22.1",
     "@uiw/react-codemirror": "^4.21.25",
     "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
-    "clsx": "^2.1.0",
+    "clsx": "^2.1.1",
     "cmdk": "^1.0.0",
     "core-js": "^3.36.1",
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
+    "dompurify": "^3.1.4",
     "exponential-backoff": "^3.1.1",
-    "js-tiktoken": "^1.0.11",
+    "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.33",
+    "langchain": "^0.1.36",
     "lodash": "^4.17.21",
     "lucide-react": "^0.364.0",
-    "next": "^14.2.1",
+    "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
+    "next-themes": "^0.3.0",
     "nodemailer": "^6.9.13",
     "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "react": "18.2.0",
     "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
-    "react-hook-form": "^7.51.3",
-    "react-icons": "^5.0.1",
+    "react-hook-form": "^7.51.5",
+    "react-icons": "^5.2.1",
     "react-responsive": "^10.0.0",
     "react18-json-view": "^0.2.8-canary.6",
     "sonner": "^1.4.41",
@@ -110,30 +116,33 @@
     "tailwindcss-animate": "^1.0.7",
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.1",
+    "vaul": "^0.9.1",
     "zod": "^3.22.4"
   },
   "devDependencies": {
     "@jedmao/location": "^3.0.0",
     "@mermaid-js/mermaid-cli": "^10.7.0",
     "@playwright/test": "^1.43.1",
-    "@testing-library/jest-dom": "^6.4.2",
-    "@testing-library/react": "^15.0.2",
+    "@tailwindcss/forms": "^0.5.7",
+    "@testing-library/jest-dom": "^6.4.5",
+    "@testing-library/react": "^15.0.7",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
+    "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
-    "@types/lodash": "^4.14.202",
+    "@types/lodash": "^4.17.4",
     "@types/node": "20.10.5",
-    "@types/nodemailer": "^6.4.14",
+    "@types/nodemailer": "^6.4.15",
     "@types/react": "^18.2.79",
     "@types/react-dom": "^18.2.25",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "autoprefixer": "^10.4.19",
-    "dotenv-cli": "^7.3.0",
+    "dotenv-cli": "^7.4.2",
     "eslint": "^8.56.0",
-    "eslint-config-next": "^14.2.1",
+    "eslint-config-next": "^14.2.3",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "node-mocks-http": "^1.14.1",

web/public/.gitignore

@@ -1,2 +0,0 @@
-openapi*.yml
-postman-collection.json