chore: release v2.43.2

---------

Co-authored-by: Marlies Mayerhofer <74332854+marliessophie@users.noreply.github.com>

.env.dev.example

@@ -14,6 +14,9 @@ DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
 NEXTAUTH_URL="http://localhost:3000"
 NEXTAUTH_SECRET="secret"
 
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
 # Langfuse experimental features
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
 
@@ -43,4 +46,6 @@ REDIS_HOST="127.0.0.1"
 REDIS_PORT=6379
 REDIS_AUTH="myredissecret"
 
-LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
+LANGFUSE_WORKER_PASSWORD=mybasicauthsecret
+# openssl rand -base64 32 used only here
+ENCRYPTION_KEY=6c16874e5c0f0cc74ddec00425fa99fbe9ffbe412b7d5d906a4b00005df91403 

.env.prod.example

@@ -76,6 +76,23 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # The page size can be adjusted if needed to optimize performance
 # DB_EXPORT_PAGE_SIZE=1000
 
+# Disable logging to `events` table in db
+# ENABLE_EVENT_LOG=false
+
+
+
+### START Envs to be deprecated in Langfuse v3.0
+
+# Disable the expensive analytics queries and related features
+# LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES="true"
+
+# Customize the default datetime offset filter used in for the UI tables
+# LANGFUSE_DEFAULT_TABLE_DATETIME_OFFSET="7" # int (days)
+
+### END Envs to be deprecated in Langfuse v3.0
+
+
+
 
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
@@ -114,4 +131,10 @@ LANGFUSE_CSP_ENFORCE_HTTPS="true"
 # Crisp chat
 # NEXT_PUBLIC_CRISP_WEBSITE_ID=
 
+# Encryption
+# ENCRYPTION_KEY=
+
+# Admin API
+# ADMIN_API_KEY=
+
 ### END Langfuse Cloud Config

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -12,10 +12,24 @@ body:
   - type: textarea
     attributes:
       label: To reproduce
-      description: Describe how to reproduce your bug. Steps, code snippets, reproduction repos etc.
+      description: Describe how to reproduce your bug. Please provide detailed steps, code snippets, reproduction repos etc.
     validations:
       required: true
+  - type: textarea
+    attributes:
+      label: SDK and container versions
+      description: If you're experiencing an issue with an integration or SDK, please ensure you're using the latest version. If you're self-hosting Langfuse, check that you're running the most recent version. If updating isn't an option, please provide the specific versions you're currently using.
   - type: textarea
     attributes:
       label: Additional information
       description: Add any other information related to the bug here, screenshots if applicable.
+  - type: dropdown
+    id: contribute
+    attributes:
+      label: Are you interested to contribute a fix for this bug?
+      description: If this is a confirmed bug, the maintainers are happy to support with guidance and review.
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true

.github/workflows/pipeline.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 8.15.5
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -68,7 +68,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 8.15.5
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -109,8 +109,6 @@ jobs:
     strategy:
       matrix:
         node-version: [20]
-    env:
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -120,7 +118,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 8.15.5
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -159,7 +157,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 8.15.5
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -214,9 +212,6 @@ jobs:
     if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
     environment: "protected branches"
     runs-on: ubuntu-latest
-    env:
-      REGISTRY: ghcr.io
-      IMAGE_NAME: ${{ github.repository }}
     permissions:
       packages: write
       contents: read
@@ -224,7 +219,7 @@ jobs:
     steps:
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 8.15.5
       - name: Setup node
         uses: actions/setup-node@v3
         with:
@@ -234,19 +229,26 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Log in to the Container registry
+      - name: Log in to the GitHub Container registry
         uses: docker/login-action@v2
         with:
-          registry: ${{ env.REGISTRY }}
+          registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta-web
         uses: docker/metadata-action@v4
         with:
           images: |
-            ${{ env.REGISTRY }}/${{ github.repository }}
+            ghcr.io/langfuse/langfuse # GitHub
+            langfuse/langfuse # Docker Hub
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -269,7 +271,8 @@ jobs:
         uses: docker/metadata-action@v4
         with:
           images: |
-            ${{ env.REGISTRY }}/${{ github.repository }}-worker
+            ghcr.io/langfuse/langfuse-worker # GitHub
+            langfuse/langfuse-worker # Docker Hub
           tags: |
             type=ref,event=branch
             type=ref,event=pr

CONTRIBUTING.md

@@ -1,3 +1,5 @@
+![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
+
 # Contributing to Langfuse
 
 First off, thanks for taking the time to contribute! ❤️
@@ -51,53 +53,63 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 
 ### Architecture Overview
 
+> [!NOTE]
+> Infrastructure will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
+> `langfuse/langfuse/worker` is under active development and not recommended for production use in Langfuse 2.x
+
 ```mermaid
 flowchart TB
-   subgraph s4["Clients"]
-      subgraph s2["langfuse/langfuse-python"]
-         Python["Python low-level SDK"]
-         Decorator["observe() decorator"] -->|extends| Python
-         OAI["OpenAI drop-in replacement"] -->|extends| Python
-         Llamaindex["LlamaIndex Integration"] -->|extends| Python
-         LCPYTHON["Langchain Python Integration"] -->|extends| Python
-         Langflow -->|uses| LCPYTHON
-         LiteLLM -->|uses| Python
-      end
-      subgraph s3["langfuse/langfuse-js"]
-         JS["JS SDK"]
-         LCJS["Langchain JS Integration"]  -->|extends| JS
-         Flowise -->|uses| LCJS
-      end
-   end
+    subgraph s4["Clients"]
+        subgraph s2["langfuse/langfuse-python"]
+            Python["Python low-level SDK"]
+            Decorator["observe() decorator"] -->|extends| Python
+            OAI["OpenAI drop-in replacement"] -->|extends| Python
+            Llamaindex["LlamaIndex Integration"] -->|extends| Python
+            LCPYTHON["Langchain Python Integration"] -->|extends| Python
+            Langflow -->|uses| LCPYTHON
+            LiteLLM -->|uses| Python
+        end
+        subgraph s3["langfuse/langfuse-js"]
+            JS["JS SDK"]
+            LCJS["Langchain JS Integration"]  -->|extends| JS
+            Flowise -->|uses| LCJS
+        end
+    end
 
-   DB[Postgres Database]
-	subgraph s1["Application (langfuse/langfuse)"]
-      API[Public HTTP API]
-      G[TRPC API]
-      I[NextAuth]
-      H[React Frontend]
-      Prisma[Prisma ORM]
-      H --> G
-      H --> I
-      G --> I
-      G --- Prisma
-      API --- Prisma
-      I --- Prisma
-	end
-   Prisma --- DB
-   JS --- API
-   Python --- API
+    DB[Postgres Database]
+    Redis[Redis]
+
+    subgraph s1["Application (langfuse/langfuse/web)"]
+        API[Public HTTP API]
+        G[TRPC API]
+        I[NextAuth]
+        H[React Frontend]
+        Prisma[Prisma ORM]
+        H --> G
+        H --> I
+        G --> I
+        G --- Prisma
+        API --- Prisma
+        I --- Prisma
+    end
+
+    subgraph s5["Application (langfuse/langfuse/worker)"]
+        Worker_API[Public HTTP API]
+    end
+
+    API --> Worker_API
+    Worker_API --- DB
+    Worker_API --- Redis
+
+    Prisma --- DB
+    JS --- API
+    Python --- API
 ```
 
-### Database Overview
+### Network Overview
 
-The diagram below may not show all relationships if the foreign key is not defined in the database schema. For instance, `trace_id` in the `observation` table is not defined as a foreign key to the `trace` table to allow unordered ingestion of these objects, but it is still a foreign key in the application code.
-
-Full database schema: [packages/shared/prisma/schema.prisma](packages/shared/prisma/schema.prisma)
-
-<img src="./packages/shared/prisma/database.svg">
-
-### Infrastructure & Network Overview
+> [!NOTE]
+> This will change in Langfuse version 3.0. More in the [GitHub Discussions](https://github.com/orgs/langfuse/discussions/1902).
 
 ```mermaid
 flowchart LR
@@ -112,15 +124,25 @@ flowchart LR
    App --- DB
 ```
 
+### Database Overview
+
+The diagram below may not show all relationships if the foreign key is not defined in the database schema. For instance, `trace_id` in the `observation` table is not defined as a foreign key to the `trace` table to allow unordered ingestion of these objects, but it is still a foreign key in the application code.
+
+Full database schema: [packages/shared/prisma/schema.prisma](packages/shared/prisma/schema.prisma)
+
+<img src="./packages/shared/prisma/database.svg">
+
 ## Repository Structure
 
 We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https://turbo.build/repo/docs) to manage the dependencies and build process. The monorepo contains the following packages:
 
 - `web`: is the main application package providing Frontend and Backend APIs for Langfuse.
 - `worker` (no production yet): contains an application for asynchronous processing of tasks. This package is not yet used in production.
-- `shared`: contains shared code between the above packages.
-- `config-eslint`: contains eslint configurations which are shared between the above packages.
-- `config-typescript`: contains typescript configurations which are shared between the above packages.
+- `packages`:
+  - `shared`: contains shared code between the above packages.
+  - `config-eslint`: contains eslint configurations which are shared between the above packages.
+  - `config-typescript`: contains typescript configurations which are shared between the above packages.
+- `ee`: contains all enterprise features. See [EE README](ee/README.md) for more details.
 
 ## Development Setup
 
@@ -131,7 +153,7 @@ Requirements
 
 **Steps**
 
-1. Fork the the repository and clone it locally
+1. Fork the repository and clone it locally
 2. Run the development database
 
    ```bash
@@ -268,6 +290,16 @@ When a new release is tagged on the `main` branch (excluding prereleases), it tr
 1. The Docker image is published to GitHub Packages with the version number and `latest` tag.
 2. The deployment is carried out on Langfuse Cloud. This is done by force pushing the `main` branch to the `production` branch during every release, using the [`release.yml`](.github/workflows/release.yml) GitHub Action.
 
+## Using secrets stored in 1Password
+
+When applying changes to non-local environments, you may need to use secrets stored in 1Password. We use the 1Password CLI for this purpose.
+
+Example:
+
+```bash
+op run --env-file="./.env" -- pnpm --filter=shared run db:deploy
+```
+
 ## License
 
 Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

LICENSE

@@ -1,10 +1,10 @@
-Copyright (c) 2023 Finto Technologies GmbH
+Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 
-* All content that resides under the "ee/" directory of this repository, if that directory exists, is licensed under the license defined in "ee/LICENSE".
-* All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
-* Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
+- All content that resides under the "ee/" and/or "web/src/ee" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
+- Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -22,4 +22,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

README.md

@@ -1,25 +1,19 @@
+![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
+
 <div align="center">
-   <a href="https://langfuse.com">
-      <h1>🪢 Langfuse</h1>
-   </a>
- <div>
-      <h3> <a href="https://cloud.langfuse.com">
-         <strong>Sign up</strong>
-       </a> · 
-      <a href="https://langfuse.com/docs/deployment/self-host">
-         <strong>Self Host Langfuse</strong>
-      </a> · 
-      <a href="https://langfuse.com/demo">
-         <strong>Demo Project (live data)</strong>
-      </a>
+   <div>
+      <h3>
+         <a href="https://cloud.langfuse.com">
+            <strong>Sign up</strong>
+         </a> · 
+         <a href="https://langfuse.com/docs/deployment/self-host">
+            <strong>Self Host</strong>
+         </a> · 
+         <a href="https://langfuse.com/demo">
+            <strong>Demo (live data)</strong>
+         </a>
       </h3>
-   <h3>
-      Langfuse is the open source LLM engineering platform.   
-   </h3>
-    <div>
-     Debug, analyze and iterate - together
    </div>
-   </br>
    <div>
       <a href="https://langfuse.com/docs"><strong>Docs</strong></a> ·
       <a href="https://langfuse.com/issues"><strong>Report Bug</strong></a> ·
@@ -28,8 +22,11 @@
       <a href="https://langfuse.com/roadmap"><strong>Roadmap</strong></a> ·
       <a href="https://langfuse.com/discord"><strong>Discord</strong></a> 
    </div>
-   </br>
-      <img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License">
+   <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
+   <br/>
+   <br/>
+   <div>
+      <a href="https://github.com/langfuse/langfuse/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License"></a>
       <a href="https://www.ycombinator.com/companies/langfuse"><img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23"></a>
       <a href="https://github.com/langfuse/langfuse/pkgs/container/langfuse"><img alt="Docker Image" src="https://img.shields.io/badge/docker-langfuse-blue?logo=Docker&logoColor=white&style=flat-square"></a>
       <a href="https://www.npmjs.com/package/langfuse"><img src="https://img.shields.io/npm/v/langfuse?style=flat-square&label=npm+langfuse" alt="langfuse npm package"></a>
@@ -40,17 +37,22 @@
 
 ## Overview
 
+_Unmute video for voice-over_
+
+https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6370149a8
+
 ### Develop
 
 - **Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
 - **Langfuse UI:** Inspect and debug complex logs ([Demo](https://langfuse.com/docs/demo), [Tracing](https://langfuse.com/docs/tracing))
-- **Prompts:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts))
+- **Prompt Management:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts/get-started))
+- **Prompt Engineering:** Test and iterate on your prompts with the [LLM Playground](https://langfuse.com/docs/playground)
 
 ### Monitor
 
 - **Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
 - **Evals:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
-  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals))
+  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals)) within Langfuse
   - Collect user feedback ([User Feedback](https://langfuse.com/docs/scores/user-feedback))
   - Manually score observations in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
 
@@ -60,12 +62,6 @@
   - Datasets let you test expected in and output pairs and benchmark performance before deploying ([Datasets](https://langfuse.com/docs/datasets))
   - Track versions and releases in your application ([Experimentation](https://langfuse.com/docs/experimentation), [Prompt Management](https://langfuse.com/docs/prompts))
 
-### Video: Langfuse in two minutes
-
-https://github.com/langfuse/langfuse/assets/2834609/6041347a-b517-4a11-8737-93ef8f8af49f
-
-_Muted by default, enable sound for voice-over_
-
 ## Get started
 
 ### Langfuse Cloud
@@ -98,25 +94,25 @@ Templated deployments: [Railway, GCP Cloud Run, AWS Fargate, Kubernetes and othe
 
 ### API Keys
 
-You require a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
+You need a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
 
 ### Ingesting Data · Instrumenting Your Application
 
-Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
+Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python (Decorators)](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
 
 You may want to use another integration to get started quickly or implement a use case that we do not yet support. However, we recommend to migrate to the Langfuse SDKs over time to ensure performance and stability.
 
-See our the [→ Quickstart](https://langfuse.com/docs/get-started) to get started in integrating Langfuse.
+See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Langfuse.
 
 ### Integrations
 
-| Integration                                                      | Supports      | Description                                                      |
-| ---------------------------------------------------------------- | ------------- | ---------------------------------------------------------------- |
-| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility.      |
-| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python        | Automated instrumentation via drop-in replacement of OpenAI SDK. |
-| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.        |
-| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python       | Automated instrumentation via LlamaIndex callback system.        |
-| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.            |
+| Integration                                                      | Supports      | Description                                                 |
+| ---------------------------------------------------------------- | ------------- | ----------------------------------------------------------- |
+| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
+| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python, JS/TS | Automated instrumentation of OpenAI SDK.                    |
+| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.                    |
+| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python        | Automated instrumentation via LlamaIndex callback system.   |
+| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.       |
 
 External projects/packages that integrate with Langfuse:
 
@@ -140,8 +136,8 @@ External projects/packages that integrate with Langfuse:
 In order of preference the best way to communicate with us:
 
 - [GitHub Discussions](https://github.com/orgs/langfuse/discussions): Contribute [ideas](https://langfuse.com/idea) [support requests](https://github.com/orgs/langfuse/discussions/categories/support) and [report bugs](https://github.com/langfuse/langfuse/issues/new?labels=%F0%9F%90%9E%E2%9D%94+unconfirmed+bug&projects=&template=bug_report.yml&title=bug%3A+) (preferred as we create a permanent, indexed artifact for other community members)
-- [Discord](https://langfuse.com/discord): For community support and to chat directly with maintainers
-- Privately: Email contact at langfuse dot com
+- [Discord](https://langfuse.com/discord): community support
+- Privately: contact at langfuse dot com
 
 ## Contributing to Langfuse
 
@@ -151,7 +147,7 @@ In order of preference the best way to communicate with us:
 
 ## License
 
-This repository is MIT licensed, except for the `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.
+This repository is MIT licensed, except for the `ee` folders. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.
 
 ## Misc
 

ee/.eslintrc.js

@@ -0,0 +1,8 @@
+/** @type {import("eslint").Linter.Config} */
+module.exports = {
+  extends: ["@repo/eslint-config/library.js"],
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    project: true,
+  },
+};

ee/LICENSE

@@ -0,0 +1,37 @@
+Langfuse Enterprise License (the “Enterprise License”)
+Copyright (c) 2024 Langfuse GmbH ('Langfuse')
+
+With regard to the Langfuse Software:
+This software and associated documentation files (the "Software") may only be
+used in production, if you (and any entity that you represent) have agreed to,
+and are in compliance with, the Langfuse Terms of Service, available
+at https://langfuse.com/terms (the “Enterprise Terms”), or other
+agreement governing the use of the Software, as agreed by you and Langfuse,
+and otherwise have a valid Langfuse Enterprise license.
+
+Subject to the foregoing sentence, you are free to
+modify this Software and publish patches to the Software. You agree that Langfuse
+and/or its licensors (as applicable) retain all right, title and interest in and
+to all such modifications and/or patches, and all such modifications and/or
+patches may only be used, copied, modified, displayed, distributed, or otherwise
+exploited with a valid Langfuse Enterprise license.
+
+Notwithstanding the foregoing, you may copy and modify
+the Software for development and testing purposes, without requiring a
+subscription. You agree that Langfuse and/or its licensors (as applicable) retain
+all right, title and interest in and to all such modifications. You are not
+granted any other rights beyond what is expressly stated herein. Subject to the
+foregoing, it is forbidden to copy, merge, publish, distribute, sublicense,
+and/or sell the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+For all third party components incorporated into the Langfuse Software, those
+components are licensed under the original license provided by the owner of the 
+applicable component.

ee/Readme.md

@@ -1,3 +1,5 @@
 # Enterprise Edition
 
-While we are currently working 100% on the open source version, we consider financing the project by developing some premium features as part of an enterprise version under a commercial license. Please reach out to us if you have specific requirements: enterprise@langfuse.com
+This folder includes features that are only available in the Enterprise Edition of Langfuse and on Langfuse Cloud.
+
+See [LICENSE](../LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

ee/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@langfuse/ee",
+  "version": "1.0.0",
+  "private": true,
+  "main": "./dist/src/index.js",
+  "types": "./dist/src/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    },
+    "./sso": {
+      "import": "./dist/src/sso/index.js",
+      "require": "./dist/src/sso/index.js"
+    }
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix"
+  },
+  "dependencies": {
+    "@langfuse/shared": "workspace:*",
+    "next": "^14.2.3",
+    "next-auth": "^4.24.7",
+    "zod": "^3.22.4"
+  },
+  "devDependencies": {
+    "@repo/eslint-config": "*",
+    "@repo/typescript-config": "*",
+    "@types/node": "^20.11.29",
+    "@typescript-eslint/parser": "^7.7.0",
+    "eslint": "^8.57.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-config-standard": "^17.1.0",
+    "eslint-plugin-prettier": "^5.1.3",
+    "prettier": "^3.2.5",
+    "ts-node": "^10.9.2",
+    "tsc-watch": "^6.2.0",
+    "typescript": "^5.4.5"
+  }
+}

ee/prettierrc.json

@@ -0,0 +1,4 @@
+{
+  "trailingComma": "es5",
+  "printWidth": 120
+}

ee/src/ee-license-check/index.ts

@@ -0,0 +1,4 @@
+import { env } from "../env";
+
+export const isEeAvailable: boolean =
+  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined;

ee/src/env.ts

@@ -0,0 +1,9 @@
+import { z } from "zod";
+import { env as sharedEnv } from "@langfuse/shared";
+
+const EnvSchema = z.object({
+  NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
+  ADMIN_API_KEY: z.string().optional(),
+});
+
+export const env = { ...sharedEnv, ...EnvSchema.parse(process.env) };

ee/src/index.ts

@@ -0,0 +1 @@
+export { isEeAvailable } from "./ee-license-check";

ee/src/sso/index.ts

@@ -0,0 +1,236 @@
+import { type Provider } from "next-auth/providers/index";
+import GoogleProvider from "next-auth/providers/google";
+import GitHubProvider from "next-auth/providers/github";
+import OktaProvider from "next-auth/providers/okta";
+import Auth0Provider from "next-auth/providers/auth0";
+import AzureADProvider from "next-auth/providers/azure-ad";
+import { isEeAvailable } from "..";
+import { prisma } from "@langfuse/shared/src/db";
+import { encrypt, decrypt } from "@langfuse/shared/encryption";
+import { SsoProviderSchema } from "./types";
+import { type NextApiRequest, type NextApiResponse } from "next";
+import { env } from "../env";
+
+// Local cache for SSO configurations
+let cachedSsoConfigs: {
+  data: SsoProviderSchema[];
+  timestamp: number;
+} | null = null;
+
+/**
+ * Get all SSO configurations from the database or from local cache and parse them into SsoProviderSchema objects.
+ *
+ * @returns {Promise<SsoProviderSchema[]>} - A list of all SSO configurations. Empty array if none are configured or EE is not available.
+ */
+async function getSsoConfigs(): Promise<SsoProviderSchema[]> {
+  if (!isEeAvailable) return [];
+  const CACHE_TTL = 60 * 1000; // 1 minute
+
+  // Set/refresh the cache if it's empty or expired
+  if (
+    cachedSsoConfigs === null ||
+    Date.now() - cachedSsoConfigs.timestamp > CACHE_TTL
+  ) {
+    // transform into zod object
+    const dbConfigs = await prisma.ssoConfig.findMany();
+    const parsedSsoConfigs = dbConfigs
+      .map((v) => {
+        try {
+          const parsedValue = SsoProviderSchema.parse(v);
+          return parsedValue;
+        } catch (e) {
+          console.error(
+            `Failed to parse SSO provider config for domain ${v.domain}`,
+            e
+          );
+          return null;
+        }
+      })
+      .filter((parsed) => parsed !== null) as SsoProviderSchema[];
+
+    cachedSsoConfigs = {
+      data: parsedSsoConfigs,
+      timestamp: Date.now(),
+    };
+  }
+
+  return cachedSsoConfigs.data;
+}
+
+/**
+ * Load all custom SSO providers from the database. To be used within `providers` in NextAuth backend configuration.
+ *
+ * @returns {Promise<Provider[]>} - A list of all custom SSO providers.
+ */
+export async function loadSsoProviders(): Promise<Provider[]> {
+  if (!isEeAvailable) return [];
+
+  const ssoConfigs = await getSsoConfigs();
+
+  const providers: Provider[] = [];
+
+  for (const dbSsoConfig of ssoConfigs) {
+    const provider = dbToNextAuthProvider(dbSsoConfig);
+    if (provider !== null) providers.push(provider);
+  }
+
+  return providers;
+}
+
+/**
+ * @returns `true` if any custom SSO provider is configured in the database.
+ */
+export async function isAnySsoConfigured(): Promise<boolean> {
+  if (!isEeAvailable) return false;
+  const ssoConfigs = await getSsoConfigs();
+  return ssoConfigs.length > 0;
+}
+
+/**
+ * Get the custom SSO providerId for a domain. To be used with NextAuth's `signIn(providerId)`.
+ *
+ * @param domain - The domain to check for a custom SSO provider, e.g. "example.com".
+ * @returns `providerId` or null if none is configured or EE is not available.
+ */
+export async function getSsoAuthProviderIdForDomain(
+  domain: string
+): Promise<string | null> {
+  if (!isEeAvailable) return null;
+  const ssoConfig = (await getSsoConfigs()).find(
+    (ssoConfig) => ssoConfig.domain === domain.toLowerCase()
+  );
+
+  if (!ssoConfig) return null;
+  return getAuthProviderIdForSsoConfig(ssoConfig);
+}
+
+/**
+ * Converts a SsoProviderConfig to a NextAuth Provider instance.
+ *
+ * @param {SsoProviderSchema} provider - The SSO configuration from the database.
+ * @returns {Provider | null} - A NextAuth Provider instance or null if parsing fails or no custom credentials are used for this SSO config.
+ */
+const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
+  // If the SsoConfig does not use custom credentials, return null as no additional provider needs to be added to NextAuth
+  if (!provider.authConfig) return null;
+
+  if (provider.authProvider === "google")
+    return GoogleProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "github")
+    return GitHubProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "auth0")
+    return Auth0Provider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "okta")
+    return OktaProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "azure-ad")
+    return AzureADProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else {
+    // Type check to ensure we handle all providers
+    // eslint-disable-next-line no-unused-vars
+    const _: never = provider;
+    throw new Error(
+      `Unrecognized SSO provider for domain ${(provider as any).domain}`
+    );
+  }
+};
+
+/**
+ * Get the custom SSO providerId for a database SSO configuration. To be used with NextAuth's `signIn(providerId)`.
+ *
+ * @param {DbSsoConfig} dbSsoConfig - The SSO configuration from the database.
+ * @returns {string} - The providerId used in NextAuth.
+ */
+const getAuthProviderIdForSsoConfig = (
+  dbSsoConfig: SsoProviderSchema
+): string => {
+  if (!dbSsoConfig.authConfig) return dbSsoConfig.authProvider;
+  return `${dbSsoConfig.domain}.${dbSsoConfig.authProvider}`;
+};
+
+export async function createNewSsoConfigHandler(
+  req: NextApiRequest,
+  res: NextApiResponse
+) {
+  try {
+    if (!isEeAvailable) {
+      res.status(403).json({ error: "EE is not available" });
+      return;
+    }
+    // allow only POST requests
+    if (req.method !== "POST") {
+      res.status(405).json({ error: "Method Not Allowed" });
+      return;
+    }
+    // check if ADMIN_API_KEY is set
+    if (!env.ADMIN_API_KEY) {
+      res.status(500).json({ error: "ADMIN_API_KEY is not set" });
+      return;
+    }
+    if (!env.ENCRYPTION_KEY) {
+      res.status(500).json({ error: "ENCRYPTION_KEY is not set" });
+      return;
+    }
+    // check bearer token
+    const { authorization } = req.headers;
+    if (!authorization) {
+      res
+        .status(401)
+        .json({ error: "Unauthorized: No authorization header provided" });
+      return;
+    }
+    const [scheme, token] = authorization.split(" ");
+    if (scheme !== "Bearer" || !token || token !== env.ADMIN_API_KEY) {
+      res.status(401).json({ error: "Unauthorized: Invalid token" });
+      return;
+    }
+
+    const body = SsoProviderSchema.safeParse(req.body);
+    if (!body.success) {
+      res.status(400).json({ error: body.error });
+      return;
+    }
+
+    const { domain, authProvider, authConfig } = body.data;
+
+    const encryptedClientSecret = authConfig
+      ? {
+          ...authConfig,
+          clientSecret: encrypt(authConfig.clientSecret),
+        }
+      : undefined;
+
+    await prisma.ssoConfig.create({
+      data: {
+        domain,
+        authProvider,
+        authConfig: encryptedClientSecret,
+      },
+    });
+    res.status(201).json({
+      message: "SSO configuration created successfully",
+    });
+  } catch (e) {
+    console.log(e);
+    res.status(500).json({ error: "Internal Server Error" });
+  }
+}

ee/src/sso/types.ts

@@ -0,0 +1,81 @@
+import { z } from "zod";
+
+const base = z.object({
+  domain: z.string().refine((v) => v === v.toLowerCase(), {
+    message: "Domain must be lowercase",
+  }),
+});
+
+export const GoogleProviderSchema = base.extend({
+  authProvider: z.literal("google"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const GithubProviderSchema = base.extend({
+  authProvider: z.literal("github"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const Auth0ProviderSchema = base.extend({
+  authProvider: z.literal("auth0"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const OktaProviderSchema = base.extend({
+  authProvider: z.literal("okta"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const AzureAdProviderSchema = base.extend({
+  authProvider: z.literal("azure-ad"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      tenantId: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
+export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
+export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
+export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
+export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+
+export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
+  GoogleProviderSchema,
+  GithubProviderSchema,
+  Auth0ProviderSchema,
+  OktaProviderSchema,
+  AzureAdProviderSchema,
+]);
+
+export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

ee/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@repo/typescript-config/base.json",
+  "compilerOptions": {
+    "moduleResolution": "NodeNext",
+    "module": "NodeNext",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "types": ["node"],
+    "target": "ES2020",
+    "rootDir": "."
+  },
+  "include": ["."],
+  "exclude": ["node_modules", "dist"]
+}

fern/apis/client/generators.yml

@@ -15,7 +15,7 @@ groups:
         version: 0.0.26
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-client
+          path: ../../../web/public/generated/api-client
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true

fern/apis/server/definition/api.yml

@@ -1,11 +1,17 @@
 name: langfuse
 docs: |
   ## Authentication
-  Authenticate with the API using Basic Auth, get API keys in the project settings:
+
+  Authenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:
 
   - username: Langfuse Public Key
   - password: Langfuse Secret Key
 
+  ## Exports
+
+  - OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
+  - Postman collection: https://cloud.langfuse.com/generated/postman/collection.json
+
 error-discrimination:
   strategy: status-code
 auth: basic

fern/apis/server/definition/commons.yml

@@ -127,6 +127,7 @@ types:
       id: string
       name: string
       description: optional<string>
+      metadata: optional<unknown>
       projectId: string
       createdAt: datetime
       updatedAt: datetime
@@ -151,6 +152,7 @@ types:
       status: DatasetStatus
       input: optional<unknown>
       expectedOutput: optional<unknown>
+      metadata: optional<unknown>
       sourceTraceId: optional<string>
       sourceObservationId: optional<string>
       datasetId: string
@@ -209,6 +211,7 @@ types:
     enum:
       - API
       - REVIEW
+      - EVAL
 
 errors:
   Error:

fern/apis/server/definition/dataset-items.yml

@@ -26,6 +26,12 @@ types:
       datasetName: string
       input: optional<unknown>
       expectedOutput: optional<unknown>
+      metadata: optional<unknown>
+      sourceTraceId: optional<string>
+      sourceObservationId: optional<string>
       id:
         type: optional<string>
         docs: Dataset items are upserted on their id
+      status:
+        type: optional<commons.DatasetStatus>
+        docs: Defaults to ACTIVE for newly created items

fern/apis/server/definition/datasets.yml

@@ -47,3 +47,4 @@ types:
     properties:
       name: string
       description: optional<string>
+      metadata: optional<unknown>

fern/apis/server/definition/metrics.yml

@@ -29,6 +29,12 @@ service:
             type: optional<string>
             allow-multiple: true
             docs: Optional filter for metrics where traces include all of these tags
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include traces on or after a certain datetime (ISO 8601)
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include traces before a certain datetime (ISO 8601)
       response: DailyMetrics
 types:
   DailyMetrics:
@@ -41,12 +47,13 @@ types:
     properties:
       date: date
       countTraces: integer
+      countObservations: integer
       totalCost: double
       usage: list<UsageByModel>
   UsageByModel:
     docs: Daily usage of a given model. Usage corresponds to the unit set for the specific model (e.g. tokens).
     properties:
-      model: string
+      model: optional<string>
       inputUsage: integer
       outputUsage: integer
       totalUsage: integer

fern/apis/server/definition/observations.yml

@@ -24,10 +24,10 @@ service:
         query-parameters:
           page:
             type: optional<integer>
-            docs: page number, starts at 1
+            docs: Page number, starts at 1.
           limit:
             type: optional<integer>
-            docs: limit of items per page
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           name: optional<string>
           userId: optional<string>
           type: optional<string>
@@ -35,7 +35,7 @@ service:
           parentObservationId: optional<string>
           fromStartTime:
             type: optional<datetime>
-            docs: Retrieve only observations with a start_time greater than this timestamp.
+            docs: Retrieve only observations with a start_time greater than this datetime (ISO 8601).
       response: ObservationsViews
 
 types:

fern/apis/server/definition/prompts.yml

@@ -4,18 +4,42 @@ imports:
   pagination: ./utils/pagination.yml
 service:
   auth: true
-  base-path: /api/public
+  base-path: /api/public/v2
   endpoints:
     get:
       docs: Get a prompt
       method: GET
+      path: /prompts/{promptName}
+      path-parameters:
+        promptName:
+          type: string
+          docs: The name of the prompt
+      request:
+        name: GetPromptRequest
+        query-parameters:
+          version:
+            type: optional<integer>
+            docs: Version of the prompt to be retrieved.
+          label:
+            type: optional<string>
+            docs: Label of the prompt to be retrieved. Defaults to "production" if no label or version is set.
+      response: Prompt
+
+    list:
+      docs: Get a list of prompt names with versions and labels
+      method: GET
       path: /prompts
       request:
-        name: GetParameterRequest
+        name: ListPromptsMetaRequest
         query-parameters:
-          name: string
-          version: optional<integer>
-      response: Prompt
+          name: optional<string>
+          label: optional<string>
+          tag: optional<string>
+          page: optional<integer>
+          limit: optional<integer>
+
+      response: PromptMetaListResponse
+
     create:
       docs: Create a prompt
       method: POST
@@ -24,6 +48,18 @@ service:
       response: Prompt
 
 types:
+  PromptMetaListResponse:
+    properties:
+      data: list<PromptMeta>
+      meta: pagination.MetaResponse
+
+  PromptMeta:
+    properties:
+      name: string
+      versions: list<integer>
+      labels: list<string>
+      tags: list<string>
+
   CreatePromptRequest:
     union:
       chat: CreateChatPromptRequest
@@ -32,20 +68,16 @@ types:
   CreateChatPromptRequest:
     properties:
       name: string
-      isActive:
-        docs: Should the prompt be promoted to production immediately?
-        type: boolean
       prompt: list<ChatMessage>
       config: optional<unknown>
+      labels: optional<list<string>>
 
   CreateTextPromptRequest:
     properties:
       name: string
-      isActive:
-        docs: Should the prompt be promoted to production immediately?
-        type: boolean
       prompt: string
       config: optional<unknown>
+      labels: optional<list<string>>
 
   Prompt:
     union:
@@ -57,6 +89,7 @@ types:
       name: string
       version: integer
       config: unknown
+      labels: list<string>
 
   ChatMessage:
     properties:

fern/apis/server/definition/score.yml

@@ -21,15 +21,24 @@ service:
         query-parameters:
           page:
             type: optional<integer>
-            docs: page number, starts at 1
+            docs: Page number, starts at 1.
           limit:
             type: optional<integer>
-            docs: limit of items per page
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           userId: optional<string>
           name: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only scores newer than this timestamp.
+            docs: Retrieve only scores newer than this datetime (ISO 8601).
+          source:
+            type: commons.ScoreSource
+            docs: Retrieve only scores from a specific source.
+          operator:
+            type: optional<string>
+            docs: Retrieve only scores with <operator> value.
+          value:
+            type: optional<double>
+            docs: Retrieve only scores with <operator> value.
       response: Scores
     get-by-id:
       docs: Get a score

fern/apis/server/definition/trace.yml

@@ -16,7 +16,7 @@ service:
           docs: The unique langfuse identifier of a trace
       response: commons.TraceWithFullDetails
     list:
-      docs: Get list of traces
+      docs: Get list of traces.
       method: GET
       path: /traces
       request:
@@ -24,15 +24,15 @@ service:
         query-parameters:
           page:
             type: optional<integer>
-            docs: page number, starts at 1
+            docs: Page number, starts at 1
           limit:
             type: optional<integer>
-            docs: limit of items per page
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           userId: optional<string>
           name: optional<string>
           fromTimestamp:
             type: optional<datetime>
-            docs: Retrieve only traces newer than this timestamp.
+            docs: Retrieve only traces newer than this datetime (ISO 8601).
           orderBy:
             type: optional<string>
             docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"

fern/apis/server/generators.yml

@@ -6,7 +6,7 @@ groups:
         version: 0.0.28
         output:
           location: local-file-system
-          path: ../../../web/generated/openapi-server
+          path: ../../../web/public/generated/api
         config:
           namespaceExport: Langfuse
           allowCustomFetcher: true
@@ -33,7 +33,7 @@ groups:
         version: 0.0.45
         output:
           location: local-file-system
-          path: ../../../generated/postman
+          path: ../../../web/public/generated/postman
   # published:
   #   generators:
   #     - name: fernapi/fern-python-sdk

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "finto",
-  "version": "0.19.30"
+  "version": "0.21.0"
 }

package.json

@@ -1,11 +1,12 @@
 {
   "name": "langfuse",
-  "version": "2.21.2",
+  "version": "2.43.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
   "engines": {
-    "node": ">=20"
+    "node": ">=20",
+    "pnpm": "<9"
   },
   "scripts": {
     "preinstall": "npx only-allow pnpm",
@@ -29,8 +30,8 @@
     "@release-it/bumper": "^6.0.1",
     "dotenv-cli": "^7.4.1",
     "prettier": "^3.2.5",
-    "release-it": "^17.1.1",
-    "turbo": "^1.13.2"
+    "release-it": "^17.2.1",
+    "turbo": "^1.13.3"
   },
   "release-it": {
     "git": {

packages/config-eslint/package.json

@@ -8,12 +8,13 @@
     "next.js"
   ],
   "devDependencies": {
-    "@vercel/style-guide": "^6.0.0",
-    "eslint-config-turbo": "^1.12.4",
-    "eslint-config-prettier": "^9.1.0",
-    "eslint-plugin-only-warn": "^1.1.0",
-    "@typescript-eslint/parser": "^7.1.0",
     "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "typescript": "^5.4.4"
+    "@typescript-eslint/parser": "^7.7.0",
+    "@vercel/style-guide": "^6.0.0",
+    "eslint-config-next": "^14.2.3",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-config-turbo": "^1.13.3",
+    "eslint-plugin-only-warn": "^1.1.0",
+    "typescript": "^5.4.5"
   }
 }

packages/shared/package.json

@@ -20,6 +20,10 @@
     "./src/server/auth": {
       "import": "./dist/src/server/auth.js",
       "require": "./dist/src/server/auth.js"
+    },
+    "./encryption": {
+      "import": "./dist/src/encryption/index.js",
+      "require": "./dist/src/encryption/index.js"
     }
   },
   "scripts": {
@@ -28,6 +32,7 @@
     "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
     "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
+    "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
     "db:reset": "dotenv -e ../../.env npx -- prisma migrate reset",
     "db:deploy": "dotenv -e ../../.env npx -- prisma migrate deploy",
     "db:seed": "dotenv -e ../../.env -- npx prisma db seed",
@@ -39,14 +44,14 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@langchain/anthropic": "^0.1.12",
-    "@langchain/core": "^0.1.55",
-    "@langchain/openai": "^0.0.26",
-    "@prisma/client": "^5.12.1",
+    "@langchain/anthropic": "^0.1.16",
+    "@langchain/core": "^0.1.61",
+    "@langchain/openai": "^0.0.28",
+    "@prisma/client": "^5.13.0",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.32",
+    "langchain": "^0.1.36",
     "prisma-extension-kysely": "^2.1.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.5"
@@ -58,7 +63,7 @@
     "@types/node": "^20.11.29",
     "@types/pg": "^8.11.5",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.2.0",
+    "@typescript-eslint/parser": "^7.7.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
@@ -66,12 +71,12 @@
     "kysely-codegen": "^0.11.0",
     "nodemon": "^3.0.3",
     "prettier": "^3.2.5",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
-    "typescript": "^5.4.4",
-    "vitest": "^1.3.1"
+    "typescript": "^5.4.5",
+    "vitest": "^1.5.3"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -4,13 +4,13 @@ export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   : ColumnType<T, T | undefined, T>;
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
-export const MembershipRole = {
+export const ProjectRole = {
     OWNER: "OWNER",
     ADMIN: "ADMIN",
     MEMBER: "MEMBER",
     VIEWER: "VIEWER"
 } as const;
-export type MembershipRole = (typeof MembershipRole)[keyof typeof MembershipRole];
+export type ProjectRole = (typeof ProjectRole)[keyof typeof ProjectRole];
 export const ObservationType = {
     SPAN: "SPAN",
     EVENT: "EVENT",
@@ -50,6 +50,11 @@ export const JobType = {
     EVAL: "EVAL"
 } as const;
 export type JobType = (typeof JobType)[keyof typeof JobType];
+export const JobConfigState = {
+    ACTIVE: "ACTIVE",
+    INACTIVE: "INACTIVE"
+} as const;
+export type JobConfigState = (typeof JobConfigState)[keyof typeof JobConfigState];
 export const JobExecutionStatus = {
     COMPLETED: "COMPLETED",
     ERROR: "ERROR",
@@ -91,7 +96,7 @@ export type AuditLog = {
     updated_at: Generated<Timestamp>;
     user_id: string;
     project_id: string;
-    user_project_role: MembershipRole;
+    user_project_role: ProjectRole;
     resource_type: string;
     resource_id: string;
     action: string;
@@ -108,6 +113,7 @@ export type Dataset = {
     id: string;
     name: string;
     description: string | null;
+    metadata: unknown | null;
     project_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
@@ -117,6 +123,7 @@ export type DatasetItem = {
     status: Generated<DatasetStatus>;
     input: unknown | null;
     expected_output: unknown | null;
+    metadata: unknown | null;
     source_trace_id: string | null;
     source_observation_id: string | null;
     dataset_id: string;
@@ -170,6 +177,7 @@ export type JobConfiguration = {
     updated_at: Generated<Timestamp>;
     project_id: string;
     job_type: JobType;
+    status: Generated<JobConfigState>;
     eval_template_id: string | null;
     score_name: string;
     filter: unknown;
@@ -191,17 +199,19 @@ export type JobExecution = {
     job_input_trace_id: string | null;
     job_output_score_id: string | null;
 };
-export type Membership = {
-    project_id: string;
-    user_id: string;
-    role: MembershipRole;
+export type LlmApiKeys = {
+    id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
+    provider: string;
+    display_secret_key: string;
+    secret_key: string;
+    project_id: string;
 };
 export type MembershipInvitation = {
     id: string;
     email: string;
-    role: MembershipRole;
+    role: ProjectRole;
     project_id: string;
     sender_id: string | null;
     created_at: Generated<Timestamp>;
@@ -284,6 +294,14 @@ export type ObservationView = {
     calculated_total_cost: string | null;
     latency: number | null;
 };
+export type PosthogIntegration = {
+    project_id: string;
+    encrypted_posthog_api_key: string;
+    posthog_host_name: string;
+    last_sync_at: Timestamp | null;
+    enabled: boolean;
+    created_at: Generated<Timestamp>;
+};
 export type Pricing = {
     id: string;
     model_name: string;
@@ -299,6 +317,13 @@ export type Project = {
     name: string;
     cloud_config: unknown | null;
 };
+export type ProjectMembership = {
+    project_id: string;
+    user_id: string;
+    role: ProjectRole;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Prompt = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -309,13 +334,15 @@ export type Prompt = {
     name: string;
     version: number;
     type: Generated<string>;
-    is_active: boolean;
+    is_active: boolean | null;
     config: Generated<unknown>;
     tags: Generated<string[]>;
+    labels: Generated<string[]>;
 };
 export type Score = {
     id: string;
     timestamp: Generated<Timestamp>;
+    project_id: string;
     name: string;
     value: number;
     source: ScoreSource;
@@ -329,6 +356,13 @@ export type Session = {
     user_id: string;
     expires: Timestamp;
 };
+export type SsoConfig = {
+    domain: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    auth_provider: string;
+    auth_config: unknown | null;
+};
 export type Trace = {
     id: string;
     external_id: string | null;
@@ -345,6 +379,8 @@ export type Trace = {
     input: unknown | null;
     output: unknown | null;
     session_id: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
 };
 export type TraceSession = {
     id: string;
@@ -402,16 +438,19 @@ export type DB = {
     events: Events;
     job_configurations: JobConfiguration;
     job_executions: JobExecution;
+    llm_api_keys: LlmApiKeys;
     membership_invitations: MembershipInvitation;
-    memberships: Membership;
     models: Model;
     observations: Observation;
     observations_view: ObservationView;
+    posthog_integrations: PosthogIntegration;
     pricings: Pricing;
+    project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
     scores: Score;
     Session: Session;
+    sso_configs: SsoConfig;
     trace_sessions: TraceSession;
     traces: Trace;
     traces_view: TraceView;

packages/shared/prisma/migrations/20240411134330_model_updates/migration.sql

@@ -0,0 +1,30 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://ai.google.dev/models/gemini#model-variations
+  -- https://cloud.google.com/vertex-ai/generative-ai/pricing
+  ('cluv2sjeo000008ih0fv23hi0', NULL, 'gemini-1.0-pro-latest', '(?i)^(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  -- stable versions priced differently
+  -- https://developers.googleblog.com/2024/02/gemini-15-available-for-private-preview-in-google-ai-studio.html
+  ('cluv2subq000108ih2mlrga6a', NULL, 'gemini-1.0-pro', '(?i)^(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2sx04000208ihbek75lsz', NULL, 'gemini-1.0-pro-001', '(?i)^(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2szw0000308ihch3n79x7', NULL, 'gemini-pro', '(?i)^(gemini-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+
+  ('cluv2t2x0000408ihfytl45l1', NULL, 'gemini-1.5-pro-latest', '(?i)^(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.0000025, 0.0000075, NULL, 'CHARACTERS', NULL, NULL),
+
+  -- https://platform.openai.com/docs/models/continuous-model-upgrades
+  ('cluv2t5k3000508ih5kve9zag', NULL, 'gpt-4-turbo-2024-04-09', '(?i)^(gpt-4-turbo-2024-04-09)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }')
+
+
+  

packages/shared/prisma/migrations/20240411194142_update_job_config_status/migration.sql

@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "JobConfigState" AS ENUM ('ACTIVE', 'INACTIVE');
+
+-- AlterTable
+ALTER TABLE "job_configurations" ADD COLUMN     "status" "JobConfigState" NOT NULL DEFAULT 'ACTIVE';

packages/shared/prisma/migrations/20240411224142_update_models/migration.sql

@@ -0,0 +1,16 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://platform.openai.com/docs/models/continuous-model-upgrades
+  ('cluvpl4ls000008l6h2gx3i07', NULL, 'gpt-4-turbo', '(?i)^(gpt-4-turbo)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }')

packages/shared/prisma/migrations/20240414203636_ee_add_sso_configs/migration.sql

@@ -0,0 +1,10 @@
+-- CreateTable
+CREATE TABLE "sso_configs" (
+    "domain" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "auth_provider" TEXT NOT NULL,
+    "auth_config" JSONB,
+
+    CONSTRAINT "sso_configs_pkey" PRIMARY KEY ("domain")
+);

packages/shared/prisma/migrations/20240415235737_index_models_model_name/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "models_model_name_idx" ON "models"("model_name");

packages/shared/prisma/migrations/20240416173813_add_internal_model_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_internal_model_idx" ON "observations"("internal_model");

packages/shared/prisma/migrations/20240417102742_metadata_on_dataset_and_dataset_item/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "dataset_items" ADD COLUMN     "metadata" JSONB;
+
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "metadata" JSONB;

packages/shared/prisma/migrations/20240419152924_posthog_integration_settings/migration.sql

@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE "posthog_integrations" (
+    "project_id" TEXT NOT NULL,
+    "encrypted_posthog_api_key" TEXT NOT NULL,
+    "posthog_host_name" TEXT NOT NULL,
+    "last_sync_at" TIMESTAMP(3),
+    "enabled" BOOLEAN NOT NULL,
+
+    CONSTRAINT "posthog_integrations_pkey" PRIMARY KEY ("project_id")
+);
+
+-- CreateIndex
+CREATE INDEX "posthog_integrations_project_id_idx" ON "posthog_integrations"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "posthog_integrations" ADD CONSTRAINT "posthog_integrations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240420134232_posthog_integration_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "posthog_integrations" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240423174013_update_models/migration.sql

@@ -0,0 +1,26 @@
+DELETE FROM models WHERE id = 'cluv2t5k3000508ih5kve9zag';
+DELETE FROM models WHERE id = 'clrkvq6iq000008ju6c16gynt';
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- updating tokenizer model to gpt-4-turbo-2024-04-09
+  ('cluv2t5k3000508ih5kve9zag', NULL, 'gpt-4-turbo-2024-04-09', '(?i)^(gpt-4-turbo-2024-04-09)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-turbo-2024-04-09" }'),
+  
+  -- update gpt-4-1106-preview naming replacing gpt-4-turbo
+  ('clrkvq6iq000008ju6c16gynt', NULL, 'gpt-4-1106-preview', '(?i)^(gpt-4-1106-preview)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }'),
+  
+  -- apparently azure supports gpt-4-preview
+  ('clv2o2x0p000008jsf9afceau', NULL, ' gpt-4-preview', '(?i)^(gpt-4-preview)$',  NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-turbo-preview" }')
+  

packages/shared/prisma/migrations/20240423192655_add_llm_api_keys/migration.sql

@@ -0,0 +1,24 @@
+-- CreateTable
+CREATE TABLE "llm_api_keys" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "provider" TEXT NOT NULL,
+    "display_secret_key" TEXT NOT NULL,
+    "secret_key" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+
+    CONSTRAINT "llm_api_keys_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_api_keys_id_key" ON "llm_api_keys"("id");
+
+-- CreateIndex
+CREATE INDEX "llm_api_keys_project_id_provider_idx" ON "llm_api_keys"("project_id", "provider");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "llm_api_keys_project_id_provider_key" ON "llm_api_keys"("project_id", "provider");
+
+-- AddForeignKey
+ALTER TABLE "llm_api_keys" ADD CONSTRAINT "llm_api_keys_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240424150909_add_job_execution_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_status_idx" ON "job_executions"("project_id", "status");

packages/shared/prisma/migrations/20240429124411_add_prompt_version_labels/migration.sql

@@ -0,0 +1,16 @@
+BEGIN;
+
+-- Step 1: Alter the 'prompts' table to add the 'labels' column
+ALTER TABLE prompts
+ADD COLUMN labels TEXT[] DEFAULT ARRAY[]::TEXT[];
+
+-- Step 2: Update the 'labels' column to include 'production' for active prompts
+UPDATE prompts
+SET labels = array_append(labels, 'production')
+WHERE is_active = TRUE;
+
+-- Step 3: Drop the required constraint on 'is_active' column.
+ALTER TABLE prompts
+ALTER COLUMN is_active DROP NOT NULL;
+
+COMMIT;

packages/shared/prisma/migrations/20240429194411_add_latest_prompt_tag/migration.sql

@@ -0,0 +1,15 @@
+UPDATE
+	prompts
+SET
+	labels = array_append(labels, 'latest')
+WHERE
+	id = (
+		SELECT
+			id
+		FROM
+			prompts AS p2
+		WHERE
+			p2.name = prompts.name
+		ORDER BY
+			created_at DESC
+		LIMIT 1);

packages/shared/prisma/migrations/20240503125742_traces_add_createdat_updatedat/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "traces" ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/shared/prisma/migrations/20240503130335_traces_index_created_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_created_at_idx" ON "traces"("created_at");

packages/shared/prisma/migrations/20240503130520_traces_index_updated_at/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+
+CREATE INDEX CONCURRENTLY "traces_updated_at_idx" ON "traces"("updated_at");

packages/shared/prisma/migrations/20240508132621_scores_add_project_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "scores" ADD COLUMN     "project_id" TEXT;

packages/shared/prisma/migrations/20240508132735_scores_add_projectid_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "scores_project_id_idx" ON "scores"("project_id");

packages/shared/prisma/migrations/20240508132736_scores_backfill_project_id/migration.sql

@@ -0,0 +1,5 @@
+-- Backfill project_id on existing scores based on linked trace_id
+UPDATE scores
+SET project_id = traces.project_id
+FROM traces 
+WHERE scores.trace_id = traces.id AND scores.project_id IS NULL;

packages/shared/prisma/migrations/20240512151529_rename_memberships_to_project_memberships/migration.sql

@@ -0,0 +1,15 @@
+-- Rename table, will cause a downtime as it needs to be locked and synced with TRPC API
+ALTER TABLE "memberships"
+RENAME TO "project_memberships";
+
+-- AlterTable
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_pkey" TO "project_memberships_pkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_project_id_fkey" TO "project_memberships_project_id_fkey";
+
+-- RenameForeignKey
+ALTER TABLE "project_memberships" RENAME CONSTRAINT "memberships_user_id_fkey" TO "project_memberships_user_id_fkey";
+
+-- RenameIndex
+ALTER INDEX "memberships_user_id_idx" RENAME TO "project_memberships_user_id_idx";

packages/shared/prisma/migrations/20240512155020_rename_enum_membership_role_to_project_role/migration.sql

@@ -0,0 +1,37 @@
+-- Goal: Rename the enum `MembershipRole` to `ProjectRole` and update all tables that use it
+
+-- Create a new Enum
+CREATE TYPE "ProjectRole" AS ENUM ('OWNER', 'ADMIN', 'MEMBER', 'VIEWER');
+
+-- Add a temporary column to tables (for each table)
+ALTER TABLE "audit_logs" ADD COLUMN "user_project_role_temp" text;
+ALTER TABLE "membership_invitations" ADD COLUMN "role_temp" text;
+ALTER TABLE "project_memberships" ADD COLUMN "role_temp" text;
+
+-- Copy data over to temporary columns (for each table)
+UPDATE "audit_logs" SET "user_project_role_temp" = "user_project_role"::text;
+UPDATE "membership_invitations" SET "role_temp" = "role"::text;
+UPDATE "project_memberships" SET "role_temp" = "role"::text;
+
+-- Drop old columns
+ALTER TABLE "audit_logs" DROP COLUMN "user_project_role";
+ALTER TABLE "membership_invitations" DROP COLUMN "role";
+ALTER TABLE "project_memberships" DROP COLUMN "role";
+
+-- Rename temporary columns to old column names
+ALTER TABLE "audit_logs" RENAME COLUMN "user_project_role_temp" TO "user_project_role";
+ALTER TABLE "membership_invitations" RENAME COLUMN "role_temp" TO "role";
+ALTER TABLE "project_memberships" RENAME COLUMN "role_temp" TO "role";
+
+-- Convert the text columns to enum columns (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" TYPE "ProjectRole" USING "user_project_role"::"ProjectRole";
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+ALTER TABLE "project_memberships" ALTER COLUMN "role" TYPE "ProjectRole" USING "role"::"ProjectRole";
+
+-- Make the columns NOT NULL (for each table)
+ALTER TABLE "audit_logs" ALTER COLUMN "user_project_role" SET NOT NULL;
+ALTER TABLE "membership_invitations" ALTER COLUMN "role" SET NOT NULL;
+ALTER TABLE "project_memberships" ALTER COLUMN "role" SET NOT NULL;
+
+-- Now finally, you can drop your old enum
+DROP TYPE "MembershipRole";

packages/shared/prisma/migrations/20240512155021_add_pricing_gpt4o/migration.sql

@@ -0,0 +1,20 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- gpt-4o
+  ('b9854a5c92dc496b997d99d20', NULL, 'gpt-4o', '(?i)^(gpt-4o)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }'),
+  
+  -- gpt-4o-2024-05-13
+  ('b9854a5c92dc496b997d99d21', NULL, 'gpt-4o-2024-05-13', '(?i)^(gpt-4o-2024-05-13)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o-2024-05-13" }')
+  

packages/shared/prisma/migrations/20240512155021_scores_drop_fk_on_traces_and_observations/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_observation_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "scores" DROP CONSTRAINT "scores_trace_id_fkey";

packages/shared/prisma/migrations/20240512155022_scores_non_null_and_add_fk_project_id/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - Made the column `project_id` on table `scores` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "scores" ALTER COLUMN "project_id" SET NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "scores" ADD CONSTRAINT "scores_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240513082203_scores_unique_id_and_projectid_instead_of_id_and_traceid/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- DropIndex
+DROP INDEX "scores_id_trace_id_key";

packages/shared/prisma/migrations/20240513082204_scores_unique_id_and_projectid_instead_of_id_and_traceid_index/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[id,project_id]` on the table `scores` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- CreateIndex
+CREATE UNIQUE INDEX CONCURRENTLY "scores_id_project_id_key" ON "scores"("id", "project_id");

packages/shared/prisma/schema.prisma

@@ -3,7 +3,7 @@
 
 generator client {
     provider        = "prisma-client-js"
-    previewFeatures = ["tracing", "views"]
+    previewFeatures = ["tracing", "views", "relationJoins"]
 }
 
 datasource db {
@@ -63,21 +63,21 @@ model Session {
 }
 
 model User {
-    id            String                 @id @default(cuid())
-    name          String?
-    email         String?                @unique
-    emailVerified DateTime?              @map("email_verified")
-    password      String?
-    image         String?
-    admin         Boolean                @default(false)
-    accounts      Account[]
-    sessions      Session[]
-    memberships   Membership[]
-    invitations   MembershipInvitation[]
-    createdAt     DateTime               @default(now()) @map("created_at")
-    updatedAt     DateTime               @default(now()) @updatedAt @map("updated_at")
-    featureFlags  String[]               @default([]) @map("feature_flags")
-    AuditLog      AuditLog[]
+    id                 String                 @id @default(cuid())
+    name               String?
+    email              String?                @unique
+    emailVerified      DateTime?              @map("email_verified")
+    password           String?
+    image              String?
+    admin              Boolean                @default(false)
+    accounts           Account[]
+    sessions           Session[]
+    projectMemberships ProjectMembership[]
+    invitations        MembershipInvitation[]
+    createdAt          DateTime               @default(now()) @map("created_at")
+    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
+    featureFlags       String[]               @default([]) @map("feature_flags")
+    AuditLog           AuditLog[]
 
     @@map("users")
 }
@@ -92,25 +92,28 @@ model VerificationToken {
 }
 
 model Project {
-    id               String                 @id @default(cuid())
-    createdAt        DateTime               @default(now()) @map("created_at")
-    updatedAt        DateTime               @default(now()) @updatedAt @map("updated_at")
-    name             String
-    cloudConfig      Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members          Membership[]
-    traces           Trace[]
-    observations     Observation[]
-    apiKeys          ApiKey[]
-    dataset          Dataset[]
-    RawEvents        Events[]
-    invitations      MembershipInvitation[]
-    sessions         TraceSession[]
-    Prompt           Prompt[]
-    Model            Model[]
-    AuditLog         AuditLog[]
-    EvalTemplate     EvalTemplate[]
-    JobConfiguration JobConfiguration[]
-    JobExecution     JobExecution[]
+    id                 String                 @id @default(cuid())
+    createdAt          DateTime               @default(now()) @map("created_at")
+    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
+    name               String
+    cloudConfig        Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
+    projectMembers     ProjectMembership[]
+    traces             Trace[]
+    observations       Observation[]
+    apiKeys            ApiKey[]
+    dataset            Dataset[]
+    RawEvents          Events[]
+    invitations        MembershipInvitation[]
+    sessions           TraceSession[]
+    Prompt             Prompt[]
+    Model              Model[]
+    AuditLog           AuditLog[]
+    EvalTemplate       EvalTemplate[]
+    JobConfiguration   JobConfiguration[]
+    JobExecution       JobExecution[]
+    LlmApiKeys         LlmApiKeys[]
+    PosthogIntegration PosthogIntegration[]
+    Score              Score[]
 
     @@map("projects")
 }
@@ -135,37 +138,54 @@ model ApiKey {
     @@map("api_keys")
 }
 
-model Membership {
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userId    String         @map("user_id")
-    user      User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    role      MembershipRole
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+model LlmApiKeys {
+    id        String   @id @unique @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    provider         String
+    displaySecretKey String @map("display_secret_key")
+    secretKey        String @map("secret_key")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    @@unique([projectId, provider])
+    @@index([projectId, provider])
+    @@map("llm_api_keys")
+}
+
+model ProjectMembership {
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId    String      @map("user_id")
+    user      User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role      ProjectRole
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@id([projectId, userId])
     @@index([userId])
-    @@map("memberships")
+    @@map("project_memberships")
 }
 
 model MembershipInvitation {
-    id        String         @id @unique @default(cuid())
+    id        String      @id @unique @default(cuid())
     email     String
-    role      MembershipRole
-    projectId String         @map("project_id")
-    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    senderId  String?        @map("sender_id")
-    sender    User?          @relation(fields: [senderId], references: [id], onDelete: SetNull)
-    createdAt DateTime       @default(now()) @map("created_at")
-    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+    role      ProjectRole
+    projectId String      @map("project_id")
+    project   Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    senderId  String?     @map("sender_id")
+    sender    User?       @relation(fields: [senderId], references: [id], onDelete: SetNull)
+    createdAt DateTime    @default(now()) @map("created_at")
+    updatedAt DateTime    @default(now()) @updatedAt @map("updated_at")
 
     @@index([projectId])
     @@index([email])
     @@map("membership_invitations")
 }
 
-enum MembershipRole {
+enum ProjectRole {
     OWNER
     ADMIN
     MEMBER
@@ -208,8 +228,9 @@ model Trace {
     output     Json?
     sessionId  String?       @map("session_id")
     session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
+    createdAt  DateTime      @default(now()) @map("created_at")
+    updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-    scores          Score[]
     DatasetRunItems DatasetRunItems[]
     DatasetItem     DatasetItem[]
     JobExecution    JobExecution[]
@@ -222,6 +243,8 @@ model Trace {
     @@index([externalId])
     @@index(timestamp)
     @@index(release)
+    @@index(updatedAt)
+    @@index(createdAt)
     @@index([tags(ops: ArrayOps)], type: Gin)
     @@map("traces")
 }
@@ -253,6 +276,8 @@ view TraceView {
 }
 
 // Update ObservationView below when making changes to this model!
+// traceId is optional only due to timing during data injestion 
+// (traceId is not necessarily known at the time of observation creation)
 model Observation {
     id                  String           @id @default(cuid())
     traceId             String?          @map("trace_id")
@@ -282,7 +307,6 @@ model Observation {
     outputCost          Decimal?          @map("output_cost")
     totalCost           Decimal?          @map("total_cost")
     completionStartTime DateTime?         @map("completion_start_time")
-    scores              Score[]
     project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
     derivedDatasetItems DatasetItem[]
     datasetRunItems     DatasetRunItems[]
@@ -302,6 +326,7 @@ model Observation {
     @@index(projectId)
     @@index(parentObservationId)
     @@index(model)
+    @@index(internalModel)
     @@index(promptId)
     @@index([projectId, startTime, type])
     @@map("observations")
@@ -368,19 +393,20 @@ enum ObservationLevel {
 model Score {
     id            String         @id @default(cuid())
     timestamp     DateTime       @default(now())
+    projectId     String         @map("project_id")
+    project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
     name          String
     value         Float
     source        ScoreSource
     comment       String?
     traceId       String         @map("trace_id")
-    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
     observationId String?        @map("observation_id")
-    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
     JobExecution  JobExecution[]
 
-    @@unique([id, traceId]) // used for upsert
+    @@unique([id, projectId]) // used for upserts via prisma
     @@index(timestamp)
     @@index([value])
+    @@index([projectId])
     @@index([traceId], type: Hash)
     @@index([observationId], type: Hash)
     @@index([source])
@@ -429,6 +455,7 @@ model Dataset {
     id           String        @id @default(cuid())
     name         String
     description  String?
+    metadata     Json?
     projectId    String        @map("project_id")
     project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
     createdAt    DateTime      @default(now()) @map("created_at")
@@ -446,6 +473,7 @@ model DatasetItem {
     status              DatasetStatus     @default(ACTIVE)
     input               Json?
     expectedOutput      Json?             @map("expected_output")
+    metadata            Json?
     sourceTraceId       String?           @map("source_trace_id")
     sourceTrace         Trace?            @relation(fields: [sourceTraceId], references: [id], onDelete: SetNull)
     sourceObservationId String?           @map("source_observation_id")
@@ -531,9 +559,10 @@ model Prompt {
     name        String
     version     Int
     type        String        @default("text")
-    isActive    Boolean       @map("is_active")
+    isActive    Boolean?      @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
     config      Json          @default("{}")
     tags        String[]      @default([])
+    labels      String[]      @default([])
     Observation Observation[]
 
     @@unique([projectId, name, version])
@@ -566,20 +595,21 @@ model Model {
     @@unique([projectId, modelName, startDate, unit])
     @@index([projectId, modelName, startDate, unit])
     @@index([projectId, modelName])
+    @@index(modelName)
     @@map("models")
 }
 
 model AuditLog {
-    id              String         @id @default(cuid())
-    createdAt       DateTime       @default(now()) @map("created_at")
-    updatedAt       DateTime       @default(now()) @updatedAt @map("updated_at")
-    userId          String         @map("user_id")
-    user            User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-    projectId       String         @map("project_id")
-    project         Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    userProjectRole MembershipRole @map("user_project_role")
-    resourceType    String         @map("resource_type")
-    resourceId      String         @map("resource_id")
+    id              String      @id @default(cuid())
+    createdAt       DateTime    @default(now()) @map("created_at")
+    updatedAt       DateTime    @default(now()) @updatedAt @map("updated_at")
+    userId          String      @map("user_id")
+    user            User        @relation(fields: [userId], references: [id], onDelete: Cascade)
+    projectId       String      @map("project_id")
+    project         Project     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userProjectRole ProjectRole @map("user_project_role")
+    resourceType    String      @map("resource_type")
+    resourceId      String      @map("resource_id")
     action          String
     before          String? //stringified JSON
     after           String? // stringified JSON
@@ -616,6 +646,11 @@ enum JobType {
     EVAL
 }
 
+enum JobConfigState {
+    ACTIVE
+    INACTIVE
+}
+
 model JobConfiguration {
     id        String   @id @default(cuid())
     createdAt DateTime @default(now()) @map("created_at")
@@ -624,9 +659,10 @@ model JobConfiguration {
     projectId String  @map("project_id")
     project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
-    jobType        JobType       @map("job_type")
-    evalTemplateId String?       @map("eval_template_id")
-    evalTemplate   EvalTemplate? @relation(fields: [evalTemplateId], references: [id], onDelete: SetNull)
+    jobType        JobType        @map("job_type")
+    status         JobConfigState @default(ACTIVE)
+    evalTemplateId String?        @map("eval_template_id")
+    evalTemplate   EvalTemplate?  @relation(fields: [evalTemplateId], references: [id], onDelete: SetNull)
 
     scoreName       String         @map("score_name")
     filter          Json
@@ -670,7 +706,36 @@ model JobExecution {
     jobOutputScoreId String? @map("job_output_score_id")
     score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
 
+    @@index([projectId, status])
     @@index([projectId, id])
     @@index([projectId])
     @@map("job_executions")
 }
+
+// Single Sign-On configuration for a domain
+// This feature is part of the Enterprise Edition
+model SsoConfig {
+    domain       String   @id @default(cuid()) // e.g. "google.com"
+    createdAt    DateTime @default(now()) @map("created_at")
+    updatedAt    DateTime @default(now()) @updatedAt @map("updated_at")
+    authProvider String   @map("auth_provider") // e.g. "okta", ee/sso/types.ts
+
+    authConfig Json? @map("auth_config")
+    // e.g. { "clientId": "1234", "clientSecret": "5678" }, null if credentials from env should be used
+    // secrets like clientSecret are encrypted on the application level
+
+    @@map("sso_configs")
+}
+
+model PosthogIntegration {
+    projectId              String    @id @map("project_id")
+    project                Project   @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    encryptedPosthogApiKey String    @map("encrypted_posthog_api_key")
+    posthogHostName        String    @map("posthog_host_name")
+    lastSyncAt             DateTime? @map("last_sync_at")
+    enabled                Boolean
+    createdAt              DateTime  @default(now()) @map("created_at")
+
+    @@index([projectId])
+    @@map("posthog_integrations")
+}

packages/shared/prisma/seed.ts

@@ -51,7 +51,7 @@ async function main() {
     create: {
       id: seedProjectId,
       name: "llm-app",
-      members: {
+      projectMembers: {
         create: {
           role: "OWNER",
           userId: user.id,
@@ -72,7 +72,7 @@ async function main() {
       name: "summary-prompt",
       project: { connect: { id: seedProjectId } },
       prompt: "prompt {{variable}} {{anotherVariable}}",
-      isActive: true,
+      labels: ["production", "latest"],
       version: 1,
       createdBy: "user-1",
     },
@@ -110,7 +110,7 @@ async function main() {
       create: {
         id: "239ad00f-562f-411d-af14-831c75ddd875",
         name: "demo-app",
-        members: {
+        projectMembers: {
           create: {
             role: "OWNER",
             userId: user.id,
@@ -165,6 +165,74 @@ async function main() {
 
     await uploadObjects(traces, observations, scores, sessions, events);
 
+    // add eval objects
+    const evalTemplate = await prisma.evalTemplate.upsert({
+      where: {
+        projectId_name_version: {
+          projectId: project1.id,
+          name: "toxicity-template",
+          version: 1,
+        },
+      },
+      create: {
+        projectId: project1.id,
+        name: "toxicity-template",
+        version: 1,
+        prompt:
+          "Please evaluate the toxicity of the following text {{input}} {{output}}",
+        model: "gpt-3.5-turbo",
+        vars: ["input", "output"],
+        outputSchema: {
+          score: "provide a score between 0 and 1",
+          reasoning: "one sentence reasoning for the score",
+        },
+        modelParams: {
+          temperature: 0.7,
+          outputTokenLimit: 100,
+          topP: 0.9,
+        },
+      },
+      update: {},
+    });
+
+    await prisma.jobConfiguration.upsert({
+      where: {
+        id: "toxicity-job",
+      },
+      create: {
+        id: "toxicity-job",
+        evalTemplateId: evalTemplate.id,
+        projectId: project1.id,
+        jobType: "EVAL",
+        status: "ACTIVE",
+        scoreName: "toxicity",
+        filter: [
+          {
+            type: "string",
+            value: "user",
+            column: "User ID",
+            operator: "contains",
+          },
+        ],
+        variableMapping: [
+          {
+            langfuseObject: "trace",
+            selectedColumnId: "input",
+            templateVariable: "input",
+          },
+          {
+            langfuseObject: "trace",
+            selectedColumnId: "metadata",
+            templateVariable: "output",
+          },
+        ],
+        targetObject: "trace",
+        sampling: 1,
+        delay: 5_000,
+      },
+      update: {},
+    });
+
     for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
       const dataset = await prisma.dataset.create({
         data: {
@@ -172,6 +240,7 @@ async function main() {
           description:
             datasetNumber === 0 ? "Dataset test description" : undefined,
           projectId: project2.id,
+          metadata: datasetNumber === 0 ? { key: "value" } : undefined,
         },
       });
 
@@ -200,6 +269,7 @@ async function main() {
               Math.random() > 0.3
                 ? "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both."
                 : undefined,
+            metadata: Math.random() > 0.5 ? { key: "value" } : undefined,
           },
         });
         datasetItemIds.push(datasetItem.id);
@@ -389,7 +459,7 @@ function createObjects(
     const session =
       Math.random() > 0.3
         ? {
-            id: `session-${i % 10}`,
+            id: `session-${i % 3}`,
             projectId: projectId,
           }
         : undefined;
@@ -431,6 +501,7 @@ function createObjects(
               value: Math.floor(Math.random() * 3) - 1,
               timestamp: traceTs,
               source: ScoreSource.REVIEW,
+              projectId,
             },
           ]
         : []),
@@ -442,6 +513,7 @@ function createObjects(
               value: Math.floor(Math.random() * 10) - 5,
               timestamp: traceTs,
               source: ScoreSource.API,
+              projectId,
             },
           ]
         : []),
@@ -520,7 +592,9 @@ function createObjects(
         const model = models[Math.floor(Math.random() * models.length)];
         const promptId =
           promptIds.get(projectId)![
-            Math.floor(Math.random() * promptIds.get(projectId)!.length)
+            Math.floor(
+              Math.random() * Math.floor(promptIds.get(projectId)!.length / 2)
+            )
           ];
 
         const generation = {
@@ -623,6 +697,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
         if (Math.random() > 0.6)
           scores.push({
@@ -631,6 +706,7 @@ function createObjects(
             observationId: generation.id,
             traceId: trace.id,
             source: ScoreSource.API,
+            projectId: trace.projectId,
           });
 
         for (let l = 0; l < Math.floor(Math.random() * 2); l++) {
@@ -694,7 +770,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 1 content",
       name: "Prompt 1",
       version: 1,
-      isActive: true,
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -703,7 +779,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 2 content",
       name: "Prompt 2",
       version: 1,
-      isActive: true,
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -712,7 +788,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 3 content",
       name: "Prompt 3 by API",
       version: 1,
-      isActive: true,
+      labels: ["production", "latest"],
     },
     {
       id: `prompt-${v4()}`,
@@ -721,7 +797,7 @@ async function generatePrompts(project: Project) {
       prompt: "Prompt 4 content",
       name: "Prompt 4",
       version: 1,
-      isActive: true,
+      labels: ["production", "latest"],
       tags: ["tag1", "tag2"],
     },
   ];
@@ -742,7 +818,7 @@ async function generatePrompts(project: Project) {
         prompt: prompt.prompt,
         name: prompt.name,
         version: prompt.version,
-        isActive: prompt.isActive,
+        labels: prompt.labels,
         tags: prompt.tags,
       },
       update: {
@@ -763,7 +839,6 @@ async function generatePrompts(project: Project) {
         temperature: 0.7,
       },
       version: 1,
-      isActive: false,
     },
     {
       id: `prompt-${v4()}`,
@@ -776,7 +851,7 @@ async function generatePrompts(project: Project) {
         topP: 0.9,
       },
       version: 2,
-      isActive: true,
+      labels: ["production"],
     },
     {
       id: `prompt-${v4()}`,
@@ -790,7 +865,7 @@ async function generatePrompts(project: Project) {
         frequencyPenalty: 0.5,
       },
       version: 3,
-      isActive: false,
+      labels: ["production", "latest"],
     },
   ];
 
@@ -811,7 +886,7 @@ async function generatePrompts(project: Project) {
         name: version.name,
         config: version.config,
         version: version.version,
-        isActive: version.isActive,
+        labels: version.labels,
       },
       update: {
         id: version.id,
@@ -840,7 +915,7 @@ async function generatePrompts(project: Project) {
         prompt: `${promptName} version ${i} content`,
         name: promptName,
         version: i,
-        isActive: i === 20,
+        labels: i === 20 ? ["production", "latest"] : [],
       },
       update: {
         id: promptId,

packages/shared/src/db.ts

@@ -24,28 +24,31 @@ const prismaClientSingleton = () => {
   });
 };
 
+const kyselySingleton = (prismaClient: PrismaClient) => {
+  return prismaClient.$extends(
+    kyselyExtension({
+      kysely: (driver) =>
+        new Kysely<DB>({
+          dialect: {
+            // This is where the magic happens!
+            createDriver: () => driver,
+            // Don't forget to customize these to match your database!
+            createAdapter: () => new PostgresAdapter(),
+            createIntrospector: (db) => new PostgresIntrospector(db),
+            createQueryCompiler: () => new PostgresQueryCompiler(),
+          },
+        }),
+    })
+  );
+};
 declare global {
   // eslint-disable-next-line no-var
   var prisma: undefined | ReturnType<typeof prismaClientSingleton>;
+  var kyselyPrisma: undefined | ReturnType<typeof kyselySingleton>;
 }
 
 export const prisma = globalThis.prisma ?? prismaClientSingleton();
-
-export const kyselyPrisma = prisma.$extends(
-  kyselyExtension({
-    kysely: (driver) =>
-      new Kysely<DB>({
-        dialect: {
-          // This is where the magic happens!
-          createDriver: () => driver,
-          // Don't forget to customize these to match your database!
-          createAdapter: () => new PostgresAdapter(),
-          createIntrospector: (db) => new PostgresIntrospector(db),
-          createQueryCompiler: () => new PostgresQueryCompiler(),
-        },
-      }),
-  })
-);
+export const kyselyPrisma = globalThis.kyselyPrisma ?? kyselySingleton(prisma);
 
 export * from "@prisma/client";
 

packages/shared/src/encryption/index.ts

@@ -0,0 +1,58 @@
+import crypto from "crypto";
+import { env } from "../env";
+
+const ENCRYPTION_KEY: string = env.ENCRYPTION_KEY || ""; // Must be 256 bits (32 bytes, 64 hex characters)
+const IV_LENGTH: number = 16; // For AES, this is always 16
+
+// Alternatively: openssl rand -hex 32
+export function keyGen() {
+  return crypto.randomBytes(32).toString("hex");
+}
+
+/**
+ * Encrypts the given plain text using AES-256-GCM algorithm.
+ *
+ * @param {string} plainText - The text to encrypt.
+ * @param {string} [keyHex=ENCRYPTION_KEY] - The encryption key in hexadecimal format.
+ * @returns {string} The encrypted data in hex format, including IV and authentication tag.
+ */
+export function encrypt(
+  plainText: string,
+  keyHex: string = ENCRYPTION_KEY
+): string {
+  const iv = crypto.randomBytes(IV_LENGTH); // Directly use Buffer returned by randomBytes
+  const cipher = crypto.createCipheriv(
+    "aes-256-gcm",
+    Buffer.from(keyHex, "hex"),
+    iv
+  );
+  let encrypted = cipher.update(plainText, "utf8", "hex");
+  encrypted += cipher.final("hex");
+  const authTag = cipher.getAuthTag();
+
+  // Return iv, encrypted data, and authTag as hex, combined in one line
+  return iv.toString("hex") + ":" + encrypted + ":" + authTag.toString("hex");
+}
+
+export function decrypt(text: string, keyHex: string = ENCRYPTION_KEY): string {
+  const [ivHex, encryptedHex, authTagHex] = text.split(":");
+  if (!ivHex || !encryptedHex || !authTagHex) {
+    throw new Error("Invalid or corrupted cipher format");
+  }
+
+  const iv = Buffer.from(ivHex, "hex");
+  const encryptedText = Buffer.from(encryptedHex, "hex");
+  const authTag = Buffer.from(authTagHex, "hex");
+
+  const decipher = crypto.createDecipheriv(
+    "aes-256-gcm",
+    Buffer.from(keyHex, "hex"),
+    iv
+  );
+  decipher.setAuthTag(authTag);
+
+  let decrypted = decipher.update(encryptedText, undefined, "utf8");
+  decrypted += decipher.final("utf8");
+
+  return decrypted.toString();
+}

packages/shared/src/env.ts

@@ -0,0 +1,13 @@
+import { z } from "zod";
+
+const EnvSchema = z.object({
+  ENCRYPTION_KEY: z
+    .string()
+    .length(
+      64,
+      "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32"
+    )
+    .optional(),
+});
+
+export const env = EnvSchema.parse(process.env);

packages/shared/src/errors/ApiError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class ApiError extends BaseError {
+  constructor(description = "Api call failed") {
+    super("ApiError", 500, description, true);
+  }
+}

packages/shared/src/errors/BaseError.ts

@@ -7,7 +7,7 @@ export class BaseError extends Error {
     name: string,
     httpCode: number,
     description: string,
-    isOperational: boolean,
+    isOperational: boolean
   ) {
     super(description);
     Object.setPrototypeOf(this, new.target.prototype); // restore prototype chain

packages/shared/src/errors/NotFoundError.ts

@@ -0,0 +1,7 @@
+import { BaseError } from "./BaseError";
+
+export class LangfuseNotFoundError extends BaseError {
+  constructor(description = "Not Found") {
+    super("LangfuseNotFoundError", 404, description, true);
+  }
+}

packages/shared/src/errors/index.ts

@@ -1,6 +1,7 @@
 export { BaseError } from "./BaseError";
-export { NotFoundError } from "./NotFoundError";
+export { LangfuseNotFoundError } from "./NotFoundError";
 export { ValidationError } from "./ValidationError";
 export { UnauthorizedError } from "./UnauthorizedError";
 export { ForbiddenError } from "./ForbiddenError";
 export { MethodNotAllowedError } from "./MethodNotAllowedError";
+export { ApiError } from "./ApiError";

packages/shared/src/features/evals/types.ts

@@ -1,4 +1,5 @@
 import z from "zod";
+import { ModelProvider, UIModelParams } from "../..";
 
 export const langfuseObjects = [
   "trace",
@@ -7,10 +8,13 @@ export const langfuseObjects = [
   "event",
 ] as const;
 
+// variable mapping stored in the db for eval templates
 export const variableMapping = z
   .object({
-    templateVariable: z.string(),
-    objectName: z.string().nullish(), // can be null as this is only required for langfuseObjects other than trace
+    templateVariable: z.string(), // variable name in the template
+    // name of the observation to extract the variable from
+    // not required for trace, as we only have one.
+    objectName: z.string().nullish(),
     langfuseObject: z.enum(langfuseObjects),
     selectedColumnId: z.string(),
   })
@@ -29,3 +33,90 @@ export const wipVariableMapping = z.object({
   langfuseObject: z.enum(langfuseObjects),
   selectedColumnId: z.string().nullish(),
 });
+
+const observationCols = [
+  {
+    name: "Metadata",
+    id: "metadata",
+    type: "stringObject",
+    internal: 'o."metadata"',
+  },
+  { name: "Input", id: "input", internal: 'o."input"' },
+  { name: "Output", id: "output", internal: 'o."output"' },
+];
+
+export const availableEvalVariables = [
+  {
+    id: "trace",
+    display: "Trace",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 't."metadata"',
+      },
+      { name: "Input", id: "input", internal: 't."input"' },
+      { name: "Output", id: "output", internal: 't."output"' },
+    ],
+  },
+  {
+    id: "span",
+    display: "Span",
+    availableColumns: observationCols,
+  },
+  {
+    id: "generation",
+    display: "Generation",
+    availableColumns: observationCols,
+  },
+  {
+    id: "event",
+    display: "Event",
+    availableColumns: observationCols,
+  },
+];
+
+export const evalLLMModels: UIModelParams[] = [
+  {
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-3.5-turbo", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
+  },
+  {
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4-turbo-preview", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
+  },
+  {
+    provider: { value: ModelProvider.OpenAI, enabled: true },
+    model: { value: "gpt-4o", enabled: true },
+    temperature: { value: 1, enabled: true },
+    maxTemperature: { value: 2, enabled: true },
+    max_tokens: { value: 256, enabled: true },
+    top_p: { value: 1, enabled: true },
+  },
+] as const;
+
+export const EvalModelNames = z.enum([
+  "gpt-3.5-turbo",
+  "gpt-4-turbo-preview",
+  "gpt-4o",
+]);
+
+export const OutputSchema = z.object({
+  reasoning: z.string(),
+  score: z.string(),
+});
+
+export enum EvalTargetObject {
+  Trace = "trace",
+}
+
+export const DEFAULT_TRACE_JOB_DELAY = 10_000;

packages/shared/src/index.ts

@@ -8,12 +8,19 @@ export * from "./types";
 export * from "./filterToPrisma";
 export * from "./tracesTable";
 export * from "./server/auth";
-export * from "./features/evals/types";
 export * from "./observationsTable";
+export { env } from "./env";
+
+// llm api
 export * from "./server/llm/types";
 export * from "./server/llm/fetchLLMCompletion";
-export * from "./server/evals/types";
+
+// evals
+export * from "./features/evals/types";
 
 // export db types only
 export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";
+
+// errors
+export * from "./errors/index";

packages/shared/src/interfaces/filters.ts

@@ -20,7 +20,7 @@ export const filterOperators = {
 export const timeFilter = z.object({
   column: z.string(),
   operator: z.enum(filterOperators.datetime),
-  value: z.date(),
+  value: z.coerce.date(), // coerce required to parse stringified dates from the db in evals
   type: z.literal("datetime"),
 });
 export const stringFilter = z.object({

packages/shared/src/queues.ts

@@ -1,22 +1,13 @@
 import { z } from "zod";
 
-export const QueueEnvelope = z.object({
-  timestamp: z.string().datetime({ offset: true }),
-  id: z.string(),
+export const TraceUpsertEvent = z.object({
+  projectId: z.string(),
+  traceId: z.string(),
 });
 
-export const TraceUpsertEvent = QueueEnvelope.extend({
-  data: z.object({
-    projectId: z.string(),
-    traceId: z.string(),
-  }),
-});
-
-export const EvalExecutionEvent = QueueEnvelope.extend({
-  data: z.object({
-    projectId: z.string(),
-    jobExecutionId: z.string(),
-  }),
+export const EvalExecutionEvent = z.object({
+  projectId: z.string(),
+  jobExecutionId: z.string(),
 });
 
 export enum QueueName {
@@ -31,10 +22,14 @@ export enum QueueJobs {
 
 export type TQueueJobTypes = {
   [QueueName.TraceUpsert]: {
+    timestamp: Date;
+    id: string;
     payload: z.infer<typeof TraceUpsertEvent>;
     name: QueueJobs.TraceUpsert;
   };
   [QueueName.EvaluationExecution]: {
+    timestamp: Date;
+    id: string;
     payload: z.infer<typeof EvalExecutionEvent>;
     name: QueueJobs.EvaluationExecution;
   };

packages/shared/src/server/auth.ts

@@ -15,6 +15,7 @@ export function getDisplaySecretKey(secretKey: string) {
 }
 
 export async function hashSecretKey(key: string) {
+  // legacy, uses bcrypt, transformed into hashed key upon first use
   const hashedKey = await hash(key, 11);
   return hashedKey;
 }

packages/shared/src/server/evals/types.ts

@@ -1,77 +0,0 @@
-import z from "zod";
-import { ModelProvider, observationsTableCols, tracesTableCols } from "../..";
-
-export const evalObjects = [
-  {
-    id: "trace",
-    display: "Trace",
-    availableColumns: [
-      ...tracesTableCols.map((c) => ({
-        name: c.name,
-        id: c.id,
-        internal: c.internal,
-      })),
-      { name: "Input", id: "input", internal: 't."input"' },
-      { name: "Output", id: "output", internal: 't."output"' },
-    ],
-  },
-  {
-    id: "span",
-    display: "Span",
-    availableColumns: [
-      ...observationsTableCols.map((c) => ({
-        name: c.name,
-        id: c.id,
-        internal: c.internal,
-      })),
-      { name: "Input", id: "input", internal: 'o."input"' },
-      { name: "Output", id: "output", internal: 'o."output"' },
-    ],
-  },
-  {
-    id: "generation",
-    display: "Generation",
-    availableColumns: [
-      ...observationsTableCols.map((c) => ({
-        name: c.name,
-        id: c.id,
-        internal: c.internal,
-      })),
-      { name: "Input", id: "input", internal: 'o."input"' },
-      { name: "Output", id: "output", internal: 'o."output"' },
-    ],
-  },
-  { id: "event", display: "Event", availableColumns: observationsTableCols },
-];
-
-export const evalModels = [
-  {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-3.5-turbo",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
-  },
-  {
-    provider: ModelProvider.OpenAI,
-    model: "gpt-4-turbo-preview",
-    temperature: 1,
-    maxTemperature: 2,
-    max_tokens: 256,
-    top_p: 1,
-  },
-] as const;
-
-export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
-
-export const OutputSchema = z.object({
-  reasoning: z.string(),
-  score: z.string(),
-});
-
-export enum EvalTargetObject {
-  Trace = "trace",
-}
-
-export const DEFAULT_TRACE_JOB_DELAY = 10_000;

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -4,6 +4,7 @@ import {
   HumanMessage,
   SystemMessage,
 } from "@langchain/core/messages";
+import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 import {
   BytesOutputParser,
   StringOutputParser,
@@ -25,6 +26,8 @@ type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
   functionCall?: LLMFunctionCall;
+  callbacks?: BaseCallbackHandler[];
+  apiKey?: string;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -34,14 +37,12 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: true;
-    functionCall: undefined;
   }
 ): Promise<IterableReadableStream<Uint8Array>>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
-    functionCall: undefined;
   }
 ): Promise<string>;
 
@@ -55,7 +56,8 @@ export async function fetchLLMCompletion(
 export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams
 ): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
-  const { messages, modelParams, streaming } = params;
+  // the apiKey must never be printed to the console
+  const { messages, modelParams, streaming, callbacks, apiKey } = params;
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
       return new HumanMessage(message.content);
@@ -68,20 +70,26 @@ export async function fetchLLMCompletion(
   let chatModel: ChatOpenAI | ChatAnthropic;
   if (modelParams.provider === ModelProvider.Anthropic) {
     chatModel = new ChatAnthropic({
-      anthropicApiKey: process.env.ANTHROPIC_API_KEY,
+      anthropicApiKey: apiKey,
       modelName: modelParams.model,
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
+      callbacks,
+    });
+  } else if (modelParams.provider === ModelProvider.OpenAI) {
+    chatModel = new ChatOpenAI({
+      openAIApiKey: apiKey,
+      modelName: modelParams.model,
+      temperature: modelParams.temperature,
+      maxTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks,
     });
   } else {
-    chatModel = new ChatOpenAI({
-      openAIApiKey: process.env.OPENAI_API_KEY,
-      modelName: modelParams.model,
-      temperature: modelParams.temperature,
-      maxTokens: modelParams.max_tokens,
-      topP: modelParams.top_p,
-    });
+    // eslint-disable-next-line no-unused-vars
+    const _exhaustiveCheck: never = modelParams;
+    throw new Error("This model provider is not supported.");
   }
 
   console.log("Making LLM call with params: ", modelParams);

packages/shared/src/server/llm/types.ts

@@ -13,17 +13,22 @@ export enum ModelProvider {
   Anthropic = "anthropic",
   OpenAI = "openai",
 }
+
 export enum ChatMessageRole {
   System = "system",
   User = "user",
   Assistant = "assistant",
 }
 export type ModelParams = AnthropicModelParams | OpenAIModelParams;
-export type UIModelParams = Required<
-  AnthropicModelParams | OpenAIModelParams
-> & {
-  maxTemperature: number;
+
+type RecordWithEnabledFlag<T> = {
+  [K in keyof T]: { value: T[K]; enabled: boolean };
 };
+export type UIModelParams = RecordWithEnabledFlag<
+  Required<AnthropicModelParams | OpenAIModelParams> & {
+    maxTemperature: number;
+  }
+>;
 
 // Generic config
 export type ModelConfig = z.infer<typeof ZodModelConfig>;
@@ -41,6 +46,8 @@ export type OpenAIModelParams = {
 } & ModelConfig;
 
 export const openAIModels = [
+  "gpt-4o",
+  "gpt-4o-2024-05-13",
   "gpt-4-turbo-preview",
   "gpt-4-1106-preview",
   "gpt-4-0613",

packages/shared/src/tracesTable.ts

@@ -1,8 +1,13 @@
 import { ObservationLevel } from "@prisma/client";
 import { ColumnDefinition, OptionsDefinition } from ".";
 
-export const tracesTableCols: ColumnDefinition[] = [
-  { name: "⭐️", id: "bookmarked", type: "boolean", internal: "t.bookmarked" },
+const tracesOnlyCols: ColumnDefinition[] = [
+  {
+    name: "⭐️",
+    id: "bookmarked",
+    type: "boolean",
+    internal: "t.bookmarked",
+  },
   { name: "ID", id: "id", type: "string", internal: "t.id" },
   {
     name: "Name",
@@ -24,6 +29,41 @@ export const tracesTableCols: ColumnDefinition[] = [
     type: "string",
     internal: 't."session_id"',
   },
+  {
+    name: "Metadata",
+    id: "metadata",
+    type: "stringObject",
+    internal: 't."metadata"',
+  },
+  {
+    name: "Version",
+    id: "version",
+    type: "string",
+    internal: 't."version"',
+  },
+  {
+    name: "Release",
+    id: "release",
+    type: "string",
+    internal: 't."release"',
+  },
+  {
+    name: "Level",
+    id: "level",
+    type: "stringOptions",
+    internal: '"level"',
+    options: Object.values(ObservationLevel).map((value) => ({ value })),
+  },
+  {
+    name: "Tags",
+    id: "tags",
+    type: "arrayOptions",
+    internal: 't."tags"',
+    options: [], // to be filled in at runtime
+  },
+];
+export const tracesTableCols: ColumnDefinition[] = [
+  ...tracesOnlyCols,
   {
     name: "Input Tokens",
     id: "inputTokens",
@@ -48,12 +88,7 @@ export const tracesTableCols: ColumnDefinition[] = [
     type: "number",
     internal: 'tm."totalTokens"',
   },
-  {
-    name: "Metadata",
-    id: "metadata",
-    type: "stringObject",
-    internal: 't."metadata"',
-  },
+
   {
     name: "Scores",
     id: "scores_avg",
@@ -84,34 +119,10 @@ export const tracesTableCols: ColumnDefinition[] = [
     type: "number",
     internal: '"calculatedTotalCost"',
   },
-  {
-    name: "Version",
-    id: "version",
-    type: "string",
-    internal: 't."version"',
-  },
-  {
-    name: "Release",
-    id: "release",
-    type: "string",
-    internal: 't."release"',
-  },
-  {
-    name: "Level",
-    id: "level",
-    type: "stringOptions",
-    internal: '"level"',
-    options: Object.values(ObservationLevel).map((value) => ({ value })),
-  },
-  {
-    name: "Tags",
-    id: "tags",
-    type: "arrayOptions",
-    internal: 't."tags"',
-    options: [], // to be filled in at runtime
-  },
 ];
 
+export const evalTableCols: ColumnDefinition[] = tracesOnlyCols;
+
 export type TraceOptions = {
   scores_avg: Array<string>;
   name: Array<OptionsDefinition>;
@@ -119,9 +130,10 @@ export type TraceOptions = {
 };
 
 export function tracesTableColsWithOptions(
-  options?: TraceOptions
+  options?: TraceOptions,
+  cols: ColumnDefinition[] = tracesTableCols
 ): ColumnDefinition[] {
-  return tracesTableCols.map((col) => {
+  return cols.map((col) => {
     if (col.id === "scores_avg") {
       return { ...col, keyOptions: options?.scores_avg ?? [] };
     }

packages/shared/tsconfig.json

@@ -7,7 +7,7 @@
     "outDir": "./dist",
     "types": ["node"],
     "target": "ES2020",
-    "rootDir": ".",
+    "rootDir": "."
   },
   "include": ["."],
   "exclude": ["node_modules", "dist"]

pnpm-workspace.yaml

@@ -3,3 +3,4 @@ packages:
   - "web"
   - "worker"
   - "packages/**"
+  - "ee"

web/Dockerfile

@@ -5,10 +5,11 @@ FROM node:20-alpine AS alpine
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
 
 FROM alpine AS base
-RUN npm install pnpm turbo --global
+RUN npm install turbo --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable
+RUN corepack prepare pnpm@8.15.5 --activate
 
 
 FROM base AS pruner

web/package.json

@@ -1,14 +1,13 @@
 {
   "name": "web",
-  "version": "2.21.2",
+  "version": "2.43.2",
   "private": true,
   "license": "MIT",
   "engines": {
     "node": ">=20"
   },
   "scripts": {
-    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
-    "build": "pnpm run copy-openapi-spec && INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
+    "build": "INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
     "lint": "dotenv -e ../.env -- next lint",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
@@ -23,15 +22,16 @@
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@aws-sdk/client-s3": "^3.550.0",
-    "@aws-sdk/lib-storage": "^3.550.0",
-    "@aws-sdk/s3-request-presigner": "^3.540.0",
+    "@aws-sdk/lib-storage": "^3.568.0",
+    "@aws-sdk/s3-request-presigner": "^3.554.0",
     "@codemirror/lang-json": "^6.0.1",
-    "@headlessui/react": "^1.7.18",
+    "@headlessui/react": "^1.7.19",
     "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.12",
-    "@langchain/core": "^0.1.55",
-    "@langchain/openai": "^0.0.26",
+    "@langchain/anthropic": "^0.1.16",
+    "@langchain/core": "^0.1.61",
+    "@langchain/openai": "^0.0.28",
+    "@langfuse/ee": "workspace:*",
     "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
     "@next-auth/prisma-adapter": "^1.0.7",
@@ -55,18 +55,20 @@
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.16",
-    "@react-email/render": "^0.0.12",
+    "@react-email/components": "^0.0.17",
+    "@react-email/render": "^0.0.13",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.109.0",
-    "@sentry/node": "^7.109.0",
-    "@sentry/profiling-node": "^7.109.0",
-    "@sentry/types": "^7.109.0",
+    "@sentry/nextjs": "^7.113.0",
+    "@sentry/node": "^7.113.0",
+    "@sentry/profiling-node": "^7.113.0",
+    "@sentry/types": "^7.113.0",
     "@t3-oss/env-nextjs": "^0.8.0",
+    "@tailwindcss/container-queries": "^0.1.1",
     "@tailwindcss/forms": "^0.5.7",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
+    "@tanstack/react-virtual": "^3.5.0",
     "@tremor/react": "3.11.1",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
@@ -74,34 +76,35 @@
     "@trpc/server": "^10.45.0",
     "@uiw/codemirror-theme-github": "^4.21.25",
     "@uiw/react-codemirror": "^4.21.25",
-    "ai": "^3.0.20",
+    "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
-    "clsx": "^2.1.0",
+    "clsx": "^2.1.1",
     "cmdk": "^1.0.0",
     "core-js": "^3.36.1",
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
+    "dompurify": "^3.1.2",
     "exponential-backoff": "^3.1.1",
-    "js-tiktoken": "^1.0.10",
+    "js-tiktoken": "^1.0.12",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.32",
+    "langchain": "^0.1.36",
     "lodash": "^4.17.21",
     "lucide-react": "^0.364.0",
-    "next": "^14.1.4",
+    "next": "^14.2.3",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
     "nodemailer": "^6.9.13",
-    "posthog-js": "^1.105.9",
+    "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
-    "prisma": "^5.12.1",
+    "prisma": "^5.13.0",
     "react": "18.2.0",
-    "react-day-picker": "^8.10.0",
+    "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
-    "react-hook-form": "^7.51.2",
+    "react-hook-form": "^7.51.3",
     "react-icons": "^5.0.1",
-    "react-responsive": "^9.0.2",
+    "react-responsive": "^10.0.0",
     "react18-json-view": "^0.2.8-canary.6",
     "sonner": "^1.4.41",
     "superjson": "2.2.1",
@@ -114,36 +117,37 @@
   "devDependencies": {
     "@jedmao/location": "^3.0.0",
     "@mermaid-js/mermaid-cli": "^10.7.0",
-    "@playwright/test": "^1.43.0",
+    "@playwright/test": "^1.43.1",
     "@testing-library/jest-dom": "^6.4.2",
-    "@testing-library/react": "^14.2.2",
+    "@testing-library/react": "^15.0.6",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
+    "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
     "@types/lodash": "^4.14.202",
     "@types/node": "20.10.5",
     "@types/nodemailer": "^6.4.14",
-    "@types/react": "^18.2.75",
-    "@types/react-dom": "^18.2.24",
+    "@types/react": "^18.2.79",
+    "@types/react-dom": "^18.2.25",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "@typescript-eslint/parser": "^6.21.0",
+    "@typescript-eslint/parser": "^7.7.0",
     "autoprefixer": "^10.4.19",
     "dotenv-cli": "^7.3.0",
     "eslint": "^8.56.0",
-    "eslint-config-next": "^14.1.4",
+    "eslint-config-next": "^14.2.3",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "node-mocks-http": "^1.14.1",
     "postcss": "^8.4.38",
     "prettier": "^3.2.5",
-    "prettier-plugin-tailwindcss": "^0.5.13",
+    "prettier-plugin-tailwindcss": "^0.5.14",
     "tailwindcss": "^3.4.3",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
     "tsx": "^4.7.2",
-    "typescript": "^5.4.4"
+    "typescript": "^5.4.5"
   },
   "ct3aMetadata": {
     "initVersion": "7.13.0"

web/public/.gitignore

@@ -1 +0,0 @@
-openapi*.yml

web/public/generated/api/openapi.yml

@@ -6,13 +6,23 @@ info:
     ## Authentication
 
 
-    Authenticate with the API using Basic Auth, get API keys in the project
-    settings:
+    Authenticate with the API using [Basic
+    Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API
+    keys in the project settings:
 
 
     - username: Langfuse Public Key
 
     - password: Langfuse Secret Key
+
+
+    ## Exports
+
+
+    - OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
+
+    - Postman collection:
+    https://cloud.langfuse.com/generated/postman/collection.json
 paths:
   /api/public/dataset-items:
     post:
@@ -482,6 +492,26 @@ paths:
             items:
               type: string
               nullable: true
+        - name: fromTimestamp
+          in: query
+          description: >-
+            Optional filter to only include traces on or after a certain
+            datetime (ISO 8601)
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
+        - name: toTimestamp
+          in: query
+          description: >-
+            Optional filter to only include traces before a certain datetime
+            (ISO 8601)
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
       responses:
         '200':
           description: ''
@@ -572,14 +602,16 @@ paths:
       parameters:
         - name: page
           in: query
-          description: page number, starts at 1
+          description: Page number, starts at 1.
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
-          description: limit of items per page
+          description: >-
+            Limit of items per page. If you encounter api issues due to too
+            large page sizes, try to reduce the limit.
           required: false
           schema:
             type: integer
@@ -618,7 +650,7 @@ paths:
           in: query
           description: >-
             Retrieve only observations with a start_time greater than this
-            timestamp.
+            datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -697,19 +729,100 @@ paths:
             application/json:
               schema: {}
       security: *ref_0
-  /api/public/prompts:
+  /api/public/v2/prompts/{promptName}:
     get:
       description: Get a prompt
       operationId: prompts_get
       tags:
         - Prompts
       parameters:
-        - name: name
-          in: query
+        - name: promptName
+          in: path
+          description: The name of the prompt
           required: true
           schema:
             type: string
         - name: version
+          in: query
+          description: Version of the prompt to be retrieved.
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: label
+          in: query
+          description: >-
+            Label of the prompt to be retrieved. Defaults to "production" if no
+            label or version is set.
+          required: false
+          schema:
+            type: string
+            nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Prompt'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
+  /api/public/v2/prompts:
+    get:
+      description: Get a list of prompt names with versions and labels
+      operationId: prompts_list
+      tags:
+        - Prompts
+      parameters:
+        - name: name
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: label
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: tag
+          in: query
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: page
+          in: query
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
           in: query
           required: false
           schema:
@@ -721,7 +834,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Prompt'
+                $ref: '#/components/schemas/PromptMetaListResponse'
         '400':
           description: ''
           content:
@@ -847,14 +960,16 @@ paths:
       parameters:
         - name: page
           in: query
-          description: page number, starts at 1
+          description: Page number, starts at 1.
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
-          description: limit of items per page
+          description: >-
+            Limit of items per page. If you encounter api issues due to too
+            large page sizes, try to reduce the limit.
           required: false
           schema:
             type: integer
@@ -873,12 +988,33 @@ paths:
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only scores newer than this timestamp.
+          description: Retrieve only scores newer than this datetime (ISO 8601).
           required: false
           schema:
             type: string
             format: date-time
             nullable: true
+        - name: source
+          in: query
+          description: Retrieve only scores from a specific source.
+          required: true
+          schema:
+            $ref: '#/components/schemas/ScoreSource'
+        - name: operator
+          in: query
+          description: Retrieve only scores with <operator> value.
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: value
+          in: query
+          description: Retrieve only scores with <operator> value.
+          required: false
+          schema:
+            type: number
+            format: double
+            nullable: true
       responses:
         '200':
           description: ''
@@ -1093,21 +1229,23 @@ paths:
       security: *ref_0
   /api/public/traces:
     get:
-      description: Get list of traces
+      description: Get list of traces.
       operationId: trace_list
       tags:
         - Trace
       parameters:
         - name: page
           in: query
-          description: page number, starts at 1
+          description: Page number, starts at 1
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
-          description: limit of items per page
+          description: >-
+            Limit of items per page. If you encounter api issues due to too
+            large page sizes, try to reduce the limit.
           required: false
           schema:
             type: integer
@@ -1126,7 +1264,7 @@ paths:
             nullable: true
         - name: fromTimestamp
           in: query
-          description: Retrieve only traces newer than this timestamp.
+          description: Retrieve only traces newer than this datetime (ISO 8601).
           required: false
           schema:
             type: string
@@ -1490,6 +1628,8 @@ components:
         description:
           type: string
           nullable: true
+        metadata:
+          nullable: true
         projectId:
           type: string
         createdAt:
@@ -1553,6 +1693,8 @@ components:
           nullable: true
         expectedOutput:
           nullable: true
+        metadata:
+          nullable: true
         sourceTraceId:
           type: string
           nullable: true
@@ -1685,6 +1827,7 @@ components:
       enum:
         - API
         - REVIEW
+        - EVAL
     CreateDatasetItemRequest:
       title: CreateDatasetItemRequest
       type: object
@@ -1695,10 +1838,22 @@ components:
           nullable: true
         expectedOutput:
           nullable: true
+        metadata:
+          nullable: true
+        sourceTraceId:
+          type: string
+          nullable: true
+        sourceObservationId:
+          type: string
+          nullable: true
         id:
           type: string
           nullable: true
           description: Dataset items are upserted on their id
+        status:
+          $ref: '#/components/schemas/DatasetStatus'
+          nullable: true
+          description: Defaults to ACTIVE for newly created items
       required:
         - datasetName
     CreateDatasetRunItemRequest:
@@ -1750,6 +1905,8 @@ components:
         description:
           type: string
           nullable: true
+        metadata:
+          nullable: true
       required:
         - name
     HealthResponse:
@@ -2337,6 +2494,8 @@ components:
           type: string
         countTraces:
           type: integer
+        countObservations:
+          type: integer
         totalCost:
           type: number
           format: double
@@ -2347,6 +2506,7 @@ components:
       required:
         - date
         - countTraces
+        - countObservations
         - totalCost
         - usage
     UsageByModel:
@@ -2358,6 +2518,7 @@ components:
       properties:
         model:
           type: string
+          nullable: true
         inputUsage:
           type: integer
         outputUsage:
@@ -2365,7 +2526,6 @@ components:
         totalUsage:
           type: integer
       required:
-        - model
         - inputUsage
         - outputUsage
         - totalUsage
@@ -2416,6 +2576,42 @@ components:
       required:
         - id
         - name
+    PromptMetaListResponse:
+      title: PromptMetaListResponse
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/PromptMeta'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    PromptMeta:
+      title: PromptMeta
+      type: object
+      properties:
+        name:
+          type: string
+        versions:
+          type: array
+          items:
+            type: integer
+        labels:
+          type: array
+          items:
+            type: string
+        tags:
+          type: array
+          items:
+            type: string
+      required:
+        - name
+        - versions
+        - labels
+        - tags
     CreatePromptRequest:
       title: CreatePromptRequest
       oneOf:
@@ -2447,18 +2643,19 @@ components:
       properties:
         name:
           type: string
-        isActive:
-          type: boolean
-          description: Should the prompt be promoted to production immediately?
         prompt:
           type: array
           items:
             $ref: '#/components/schemas/ChatMessage'
         config:
           nullable: true
+        labels:
+          type: array
+          items:
+            type: string
+          nullable: true
       required:
         - name
-        - isActive
         - prompt
     CreateTextPromptRequest:
       title: CreateTextPromptRequest
@@ -2466,16 +2663,17 @@ components:
       properties:
         name:
           type: string
-        isActive:
-          type: boolean
-          description: Should the prompt be promoted to production immediately?
         prompt:
           type: string
         config:
           nullable: true
+        labels:
+          type: array
+          items:
+            type: string
+          nullable: true
       required:
         - name
-        - isActive
         - prompt
     Prompt:
       title: Prompt
@@ -2511,10 +2709,15 @@ components:
         version:
           type: integer
         config: {}
+        labels:
+          type: array
+          items:
+            type: string
       required:
         - name
         - version
         - config
+        - labels
     ChatMessage:
       title: ChatMessage
       type: object

web/public/generated/postman/collection.json

@@ -2,7 +2,7 @@
   "info": {
     "name": "Langfuse",
     "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-    "description": "## Authentication\n\nAuthenticate with the API using Basic Auth, get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key"
+    "description": "## Authentication\n\nAuthenticate with the API using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key\n\n## Exports\n\n- OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml\n- Postman collection: https://cloud.langfuse.com/generated/postman/collection.json"
   },
   "variable": [
     {
@@ -65,7 +65,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"id\": \"example\"\n}",
+              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"metadata\": \"UNKNOWN\",\n    \"sourceTraceId\": \"example\",\n    \"sourceObservationId\": \"example\",\n    \"id\": \"example\",\n    \"status\": \"ACTIVE\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -245,7 +245,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"name\": \"example\",\n    \"description\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"example\",\n    \"description\": \"example\",\n    \"metadata\": \"UNKNOWN\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -379,7 +379,7 @@
           "request": {
             "description": "Get daily metrics of the Langfuse project",
             "url": {
-              "raw": "{{baseUrl}}/api/public/metrics/daily?page=&limit=&traceName=&userId=&tags=",
+              "raw": "{{baseUrl}}/api/public/metrics/daily?page=&limit=&traceName=&userId=&tags=&fromTimestamp=&toTimestamp=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -414,6 +414,16 @@
                   "key": "tags",
                   "value": "",
                   "description": "Optional filter for metrics where traces include all of these tags"
+                },
+                {
+                  "key": "fromTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include traces on or after a certain datetime (ISO 8601)"
+                },
+                {
+                  "key": "toTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include traces before a certain datetime (ISO 8601)"
                 }
               ],
               "variable": []
@@ -483,12 +493,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": "page number, starts at 1"
+                  "description": "Page number, starts at 1."
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": "limit of items per page"
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
                 },
                 {
                   "key": "name",
@@ -518,7 +528,7 @@
                 {
                   "key": "fromStartTime",
                   "value": "",
-                  "description": "Retrieve only observations with a start_time greater than this timestamp."
+                  "description": "Retrieve only observations with a start_time greater than this datetime (ISO 8601)."
                 }
               ],
               "variable": []
@@ -575,13 +585,58 @@
           "request": {
             "description": "Get a prompt",
             "url": {
-              "raw": "{{baseUrl}}/api/public/prompts?name=&version=",
+              "raw": "{{baseUrl}}/api/public/v2/prompts/:promptName?version=&label=",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
+                "prompts",
+                ":promptName"
+              ],
+              "query": [
+                {
+                  "key": "version",
+                  "value": "",
+                  "description": "Version of the prompt to be retrieved."
+                },
+                {
+                  "key": "label",
+                  "value": "",
+                  "description": "Label of the prompt to be retrieved. Defaults to \"production\" if no label or version is set."
+                }
+              ],
+              "variable": [
+                {
+                  "key": "promptName",
+                  "value": "",
+                  "description": "The name of the prompt"
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get a list of prompt names with versions and labels",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/v2/prompts?name=&label=&tag=&page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "v2",
                 "prompts"
               ],
               "query": [
@@ -591,7 +646,22 @@
                   "description": null
                 },
                 {
-                  "key": "version",
+                  "key": "label",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "tag",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "limit",
                   "value": "",
                   "description": null
                 }
@@ -611,13 +681,14 @@
           "request": {
             "description": "Create a prompt",
             "url": {
-              "raw": "{{baseUrl}}/api/public/prompts",
+              "raw": "{{baseUrl}}/api/public/v2/prompts",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
+                "v2",
                 "prompts"
               ],
               "query": [],
@@ -628,7 +699,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ]\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -684,7 +755,7 @@
           "request": {
             "description": "Get a list of scores",
             "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -697,12 +768,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": "page number, starts at 1"
+                  "description": "Page number, starts at 1."
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": "limit of items per page"
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
                 },
                 {
                   "key": "userId",
@@ -717,7 +788,22 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only scores newer than this timestamp."
+                  "description": "Retrieve only scores newer than this datetime (ISO 8601)."
+                },
+                {
+                  "key": "source",
+                  "value": "",
+                  "description": "Retrieve only scores from a specific source."
+                },
+                {
+                  "key": "operator",
+                  "value": "",
+                  "description": "Retrieve only scores with <operator> value."
+                },
+                {
+                  "key": "value",
+                  "value": "",
+                  "description": "Retrieve only scores with <operator> value."
                 }
               ],
               "variable": []
@@ -875,7 +961,7 @@
           "_type": "endpoint",
           "name": "List",
           "request": {
-            "description": "Get list of traces",
+            "description": "Get list of traces.",
             "url": {
               "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&fromTimestamp=&orderBy=&tags=",
               "host": [
@@ -890,12 +976,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": "page number, starts at 1"
+                  "description": "Page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": "limit of items per page"
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
                 },
                 {
                   "key": "userId",
@@ -910,7 +996,7 @@
                 {
                   "key": "fromTimestamp",
                   "value": "",
-                  "description": "Retrieve only traces newer than this timestamp."
+                  "description": "Retrieve only traces newer than this datetime (ISO 8601)."
                 },
                 {
                   "key": "orderBy",

web/public/images/posthog-logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 140"><g fill-rule="nonzero" fill="none"><path d="M55.383 75.225c-1.874 3.748-7.222 3.748-9.096 0l-4.482-8.963a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.963Zm0 50.836c-1.874 3.747-7.222 3.747-9.096 0l-4.482-8.964a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.964Z" fill="#1D4AFF"/><path d="M0 106.765c0-4.53 5.477-6.8 8.68-3.596l23.307 23.307c3.204 3.204.935 8.68-3.595 8.68H5.085A5.085 5.085 0 0 1 0 130.073v-23.307Zm0-24.55c0 1.35.536 2.643 1.49 3.596l47.856 47.856a5.086 5.086 0 0 0 3.595 1.49h26.286c4.53 0 6.799-5.477 3.595-8.681L8.681 52.334C5.477 49.131 0 51.4 0 55.93v26.286Zm0-50.834c0 1.348.536 2.642 1.49 3.595l98.69 98.691a5.086 5.086 0 0 0 3.596 1.49h26.285c4.53 0 6.8-5.477 3.596-8.681L8.681 1.5C5.477-1.704 0 .565 0 5.095v26.286Zm50.835 0c0 1.348.536 2.642 1.49 3.595l91.5 91.5c3.203 3.204 8.68.935 8.68-3.596V96.595a5.086 5.086 0 0 0-1.49-3.596l-91.5-91.5c-3.203-3.203-8.68-.934-8.68 3.596v26.286ZM110.35 1.5c-3.203-3.204-8.68-.935-8.68 3.595v26.286c0 1.348.536 2.642 1.49 3.595l40.664 40.665c3.204 3.204 8.68.935 8.68-3.596V45.76a5.086 5.086 0 0 0-1.489-3.596L110.35 1.5Z" fill="#F9BD2B"/><path d="m216.24 107.388-47.864-47.863c-3.204-3.204-8.681-.935-8.681 3.595v66.952a5.085 5.085 0 0 0 5.085 5.085h74.142a5.085 5.085 0 0 0 5.085-5.085v-6.097c0-2.809-2.286-5.052-5.07-5.414a39.27 39.27 0 0 1-22.698-11.173Zm-32.145 11.502a8.137 8.137 0 0 1-8.133-8.134 8.137 8.137 0 0 1 8.133-8.134 8.137 8.137 0 0 1 8.134 8.134 8.137 8.137 0 0 1-8.134 8.134Z" fill="#000"/><path d="M0 130.072a5.085 5.085 0 0 0 5.085 5.085h23.307c4.53 0 6.799-5.477 3.595-8.681L8.681 103.169C5.477 99.966 0 102.235 0 106.765v23.307Zm50.835-86.418L8.68 1.5C5.477-1.704 0 .565 0 5.095v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V43.654ZM8.68 52.334C5.477 49.131 0 51.4 0 55.93v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V94.489L8.68 52.334Z" fill="#1D4AFF"/><path d="M101.67 45.76a5.083 5.083 0 0 0-1.49-3.596L59.516 1.5c-3.204-3.204-8.681-.935-8.681 3.595v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V45.76Zm-50.835 89.397h28.392c4.53 0 6.799-5.477 3.595-8.681L50.835 94.489v40.668Zm0-91.503v38.562c0 1.348.536 2.642 1.49 3.595l49.345 49.346V96.595a5.084 5.084 0 0 0-1.49-3.596L50.835 43.654Z" fill="#F54E00"/><path d="M303.32 114.86h20.888V80.22h17.452c19.17 0 31.466-11.37 31.466-28.954 0-17.584-12.295-28.954-31.466-28.954h-38.34v92.547Zm20.888-52.488V40.16h15.337c7.932 0 12.692 4.23 12.692 11.105 0 6.876-4.76 11.106-12.692 11.106h-15.337Zm86.71 53.545c20.36 0 35.167-14.543 35.167-34.375 0-19.831-14.807-34.374-35.167-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375Zm-15.866-34.375c0-10.577 6.346-17.848 15.866-17.848 9.386 0 15.733 7.271 15.733 17.848 0 10.577-6.347 17.849-15.733 17.849-9.52 0-15.866-7.272-15.866-17.849Zm84.462 34.375c15.601 0 26.178-9.784 26.178-21.286 0-26.97-35.829-18.245-35.829-28.822 0-2.908 3.04-4.759 7.404-4.759 4.495 0 9.916 2.776 11.634 8.858l15.601-6.479c-3.04-9.65-14.279-16.261-27.896-16.261-14.676 0-23.798 8.725-23.798 19.17 0 25.252 35.3 18.245 35.3 28.69 0 3.702-3.437 6.214-8.594 6.214-7.403 0-12.56-5.156-14.146-11.37l-15.601 6.081c3.438 10.048 13.486 19.964 29.747 19.964Zm76.43-1.718-1.321-16.791c-2.248 1.19-5.157 1.586-7.536 1.586-4.76 0-7.933-3.437-7.933-9.387V64.355h16.13v-16.13h-16.13V28.923h-19.435v19.302h-10.577v16.13h10.577v27.764c0 16.13 10.974 23.798 25.384 23.798 3.967 0 7.669-.66 10.842-1.718Zm67.764-91.887v35.961h-36.755v-35.96h-20.89v92.546h20.89V76.122h36.755v38.737h21.021V22.312h-21.021Zm67.386 93.605c20.36 0 35.168-14.543 35.168-34.375 0-19.831-14.807-34.374-35.168-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375ZM675.23 81.542c0-10.577 6.346-17.848 15.865-17.848 9.387 0 15.733 7.271 15.733 17.848 0 10.577-6.346 17.849-15.733 17.849-9.519 0-15.865-7.272-15.865-17.849Zm88.545 31.202c7.272 0 13.75-2.512 17.188-6.875v6.346c0 7.404-5.95 12.56-15.072 12.56-6.479 0-12.164-3.173-13.09-8.594l-17.715 2.777c2.38 12.56 15.204 21.022 30.805 21.022 20.492 0 34.11-12.032 34.11-29.88V48.225h-19.17v5.685c-3.57-4.098-9.652-6.742-17.452-6.742-18.51 0-30.144 12.692-30.144 32.788 0 20.096 11.634 32.788 30.54 32.788ZM752.14 79.956c0-9.916 5.817-16.262 14.807-16.262 9.123 0 14.94 6.346 14.94 16.262s-5.817 16.262-14.94 16.262c-8.99 0-14.807-6.346-14.807-16.262Z" fill="#000"/></g></svg>

web/scripts/model-match.ts

@@ -62,6 +62,7 @@ export async function modelMatch() {
       where: {
         internalModel: null,
         type: "GENERATION",
+        model: { not: null },
       },
       take: BATCH_SIZE,
       skip: index * BATCH_SIZE,
@@ -235,14 +236,23 @@ export async function modelMatch() {
     }
   }
 
-  await prisma.observation.updateMany({
-    where: {
-      internalModel: "LANGFUSETMPNOMODEL",
-    },
-    data: {
-      internalModel: null,
-    },
-  });
+  let updatedCount;
+  do {
+    console.log(`Updating LANGFUSETMPNOMODEL ${updatedCount}`);
+    const result = await prisma.$queryRaw<[{ id: string }]>`
+    WITH to_update AS (
+      SELECT id 
+      FROM observations 
+      WHERE internal_model = 'LANGFUSETMPNOMODEL'
+      AND "type" = 'GENERATION'
+      LIMIT 50000
+    )
+    UPDATE observations
+    set internal_model = NULL
+    WHERE id IN (SELECT id FROM to_update)
+    RETURNING id;`;
+    updatedCount = result.length;
+  } while (updatedCount > 0);
 
   const end = Date.now();
 

web/sentry.server.config.ts

@@ -15,13 +15,12 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     tracesSampler: (samplingContext: SamplingContext) => {
       if (
         samplingContext.request &&
-        samplingContext.request.method === "POST" &&
         samplingContext.request.url &&
         samplingContext.request.url.includes("api/trpc")
       ) {
-        return 0.9;
+        return 0.3;
       }
-      return 0.15;
+      return 0.1;
     },
 
     profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate
@@ -29,6 +28,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),
       new Sentry.Integrations.Prisma({ client: prisma }),
+      Sentry.metrics.metricsAggregatorIntegration(),
     ],
 
     // filter out passwords from the signup request body

web/src/__e2e__/auth.spec.ts

@@ -10,7 +10,7 @@ test("should redirect to home if signed in", async ({ page }) => {
   await page.goto("/auth/sign-in");
   await page.fill('input[name="email"]', "demo@langfuse.com");
   await page.fill('input[type="password"]', "password");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
 
   // wait 2 seconds
   await page.waitForTimeout(2000);
@@ -32,7 +32,7 @@ test("Successfully sign up & able to go to homepage", async ({ page }) => {
   await page.fill('input[name="name"]', "demo lang");
   await page.fill('input[name="email"]', randomEmailAddress());
   await page.fill('input[type="password"]', "password2");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // see get started page
   await expect(page).toHaveURL("/?getStarted=1");
@@ -45,7 +45,7 @@ test("Successfully sign up & able to go to homepage with uppercase email", async
   await page.fill('input[name="name"]', "demo lang");
   await page.fill('input[name="email"]', "A" + randomEmailAddress());
   await page.fill('input[type="password"]', "password3");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // see get started page
   await expect(page).toHaveURL("/?getStarted=1");
@@ -55,13 +55,13 @@ test("Signup input validation", async ({ page }) => {
   await page.goto("auth/sign-up");
   await page.fill('input[name="email"]', "notanemail");
   await page.fill('input[type="password"]', "shortPw");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   await expect(page.getByText("Invalid email")).toBeVisible();
   await expect(
     page.getByText("Password must be at least 8 characters long"),
   ).toBeVisible();
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // don't see get started page
   await expect(page).not.toHaveURL("/?getStarted=1");
@@ -70,3 +70,38 @@ test("Signup input validation", async ({ page }) => {
 // random email address to be used in tests
 const randomEmailAddress = () =>
   Math.random().toString(36).substring(2, 11) + "@example.com";
+
+test("Unauthenticated user should be redirected to target URL after login", async ({
+  page,
+}) => {
+  await page.goto("/auth/sign-in");
+  await page.fill('input[name="email"]', "demo@langfuse.com");
+  await page.fill('input[type="password"]', "password");
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
+
+  // wait 2 seconds
+  await page.waitForTimeout(2000);
+
+  // project id and prompt from seed.ts
+  const promptUrl = "/project/7a88fb47-b4e2-43b8-a06c-a5ce950dc53a/prompts/summary-prompt";
+
+  await page.getByRole("button", { name: /Demo User/ }).click();
+
+  await page.getByRole("menuitem", { name: "Sign Out" }).click();
+
+  await expect(page).toHaveURL("/auth/sign-in");
+
+  await page.goto(promptUrl);
+
+  await page.waitForTimeout(2000);
+
+  await expect(page).toHaveURL(/targetPath/);
+
+  await page.fill('input[name="email"]', "demo@langfuse.com");
+  await page.fill('input[type="password"]', "password");
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
+
+  await page.waitForTimeout(2000);
+
+  await expect(page).toHaveURL(promptUrl);
+});

web/src/__e2e__/create-project.spec.ts

@@ -7,7 +7,7 @@ test("should see new projects dialog open after clicking new project btn", async
   await page.fill('input[name="name"]', "demo user");
   await page.fill('input[name="email"]', randomEmailAddress());
   await page.fill('input[type="password"]', "password");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   await page.waitForTimeout(2000);
   expect(await page.innerHTML("data-testid=create-new-project-title")).toBe(
@@ -24,7 +24,7 @@ test("Create a project with provided name", async ({ page }) => {
   await page.goto("/auth/sign-in");
   await page.fill('input[name="email"]', "demo@langfuse.com");
   await page.fill('input[type="password"]', "password");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
   await page.waitForTimeout(2000);
   await page.isVisible('[data-testid="new-project-form"]');
   await page.click('[data-testid="create-project-btn"]');

web/src/__tests__/datasets.servertest.ts

@@ -5,63 +5,12 @@ import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 } from "uuid";
 
 describe("/api/public/datasets and /api/public/dataset-items API Endpoints", () => {
-  beforeEach(async () => await pruneDatabase());
-  afterEach(async () => await pruneDatabase());
-
-  it("should create and get a dataset", async () => {
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name",
-      description: "dataset-description",
-    });
-
-    const dbDataset = await prisma.dataset.findMany({
-      where: {
-        name: "dataset-name",
-      },
-    });
-
-    expect(dbDataset.length).toBeGreaterThan(0);
-
-    const getDataset = await makeAPICall(
-      "GET",
-      `/api/public/datasets/dataset-name`,
-    );
-
-    expect(getDataset.status).toBe(200);
-    expect(getDataset.body).toMatchObject({
-      name: "dataset-name",
-      description: "dataset-description",
-    });
-  });
-
-  it("GET datasets", async () => {
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name-1",
-    });
-
-    await makeAPICall("POST", "/api/public/datasets", {
-      name: "dataset-name-2",
-    });
-
-    const datasetItemId = v4();
-
-    const createItemRes = await makeAPICall<{
-      datasetName: string; // field that can break if the API changes as it is not a db column
-    }>("POST", "/api/public/dataset-items", {
-      datasetName: "dataset-name-2",
-      input: { key: "value" },
-      expectedOutput: { key: "value" },
-      id: datasetItemId,
-    });
-
-    expect(createItemRes.status).toBe(200);
-    expect(createItemRes.body).toMatchObject({
-      datasetName: "dataset-name-2", // not included in db table
-    });
-
-    const traceId = v4();
-    const observationId = v4();
+  const traceId = v4();
+  const observationId = v4();
+  beforeEach(async () => {
+    await pruneDatabase();
 
+    // create sample trace and observation
     const response = await makeAPICall("POST", "/api/public/ingestion", {
       batch: [
         {
@@ -97,6 +46,62 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       ],
     });
     expect(response.status).toBe(207);
+  });
+  afterEach(async () => await pruneDatabase());
+
+  it("should create and get a dataset, include special characters", async () => {
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset + name",
+      description: "dataset-description",
+      metadata: { foo: "bar" },
+    });
+
+    const dbDataset = await prisma.dataset.findMany({
+      where: {
+        name: "dataset + name",
+      },
+    });
+
+    expect(dbDataset.length).toBeGreaterThan(0);
+
+    const getDataset = await makeAPICall(
+      "GET",
+      `/api/public/datasets/${encodeURIComponent("dataset + name")}`,
+    );
+
+    expect(getDataset.status).toBe(200);
+    expect(getDataset.body).toMatchObject({
+      name: "dataset + name",
+      description: "dataset-description",
+      metadata: { foo: "bar" },
+    });
+  });
+
+  it("GET datasets", async () => {
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name-1",
+    });
+
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name-2",
+    });
+
+    const datasetItemId = v4();
+
+    const createItemRes = await makeAPICall<{
+      datasetName: string; // field that can break if the API changes as it is not a db column
+    }>("POST", "/api/public/dataset-items", {
+      datasetName: "dataset-name-2",
+      input: { key: "value" },
+      expectedOutput: { key: "value" },
+      metadata: { key: "value-dataset-item" },
+      id: datasetItemId,
+    });
+
+    expect(createItemRes.status).toBe(200);
+    expect(createItemRes.body).toMatchObject({
+      datasetName: "dataset-name-2", // not included in db table
+    });
 
     await makeAPICall("POST", "/api/public/dataset-run-items", {
       datasetItemId: datasetItemId,
@@ -136,6 +141,9 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       datasetName: "dataset-name",
       input: { key: "value" },
       expectedOutput: { key: "value" },
+      metadata: { key: "value-dataset-item" },
+      sourceTraceId: traceId,
+      sourceObservationId: observationId,
     });
     const dbDatasetItem = await prisma.datasetItem.findFirst({
       where: {
@@ -159,6 +167,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
           id: dbDatasetItem!.id,
           input: { key: "value" },
           expectedOutput: { key: "value" },
+          metadata: { key: "value-dataset-item" },
           datasetName: "dataset-name", // not included in db table
         },
       ],
@@ -173,7 +182,10 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       id: dbDatasetItem!.id,
       input: { key: "value" },
       expectedOutput: { key: "value" },
+      metadata: { key: "value-dataset-item" },
       datasetName: "dataset-name", // not included in db table
+      sourceTraceId: traceId,
+      sourceObservationId: observationId,
     });
   });
 
@@ -186,6 +198,7 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       id: "dataset-item-id",
       datasetName: "dataset-name",
       input: { key: "value" },
+      metadata: { key: "value-dataset-item" },
     });
     expect(item1.status).toBe(200);
     expect(item1.body).toMatchObject({
@@ -196,11 +209,15 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       id: "dataset-item-id",
       datasetName: "dataset-name",
       input: { key: "value2" },
+      metadata: ["hello-world"],
+      status: "ARCHIVED",
     });
     expect(item2.status).toBe(200);
     expect(item2.body).toMatchObject({
       id: "dataset-item-id",
       input: { key: "value2" },
+      metadata: ["hello-world"],
+      status: "ARCHIVED",
     });
 
     const dbDatasetItem = await prisma.datasetItem.findFirst({
@@ -208,22 +225,24 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
     });
     expect(dbDatasetItem).not.toBeNull();
     expect(dbDatasetItem?.input).toMatchObject({ key: "value2" });
+    expect(dbDatasetItem?.metadata).toMatchObject(["hello-world"]);
+    expect(dbDatasetItem?.status).toBe("ARCHIVED");
   });
 
-  it("should create and get a dataset run", async () => {
+  it("should create and get a dataset run, include special characters", async () => {
     const dataset = await makeAPICall<{ id: string }>(
       "POST",
       "/api/public/datasets",
       {
-        name: "dataset-name",
+        name: "dataset name",
       },
     );
     expect(dataset.status).toBe(200);
     expect(dataset.body).toMatchObject({
-      name: "dataset-name",
+      name: "dataset name",
     });
     await makeAPICall("POST", "/api/public/dataset-items", {
-      datasetName: "dataset-name",
+      datasetName: "dataset name",
       id: "dataset-item-id",
       input: { key: "value" },
       expectedOutput: { key: "value" },
@@ -272,14 +291,14 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
       {
         datasetItemId: "dataset-item-id",
         observationId: observationId,
-        runName: "run-only-observation",
+        runName: "run + only + observation",
         runDescription: "run-description",
         metadata: { key: "value" },
       },
     );
     const dbRunObservation = await prisma.datasetRuns.findFirst({
       where: {
-        name: "run-only-observation",
+        name: "run + only + observation",
       },
       include: {
         datasetRunItems: true,
@@ -298,21 +317,22 @@ describe("/api/public/datasets and /api/public/dataset-items API Endpoints", ()
 
     const getRunAPI = await makeAPICall(
       "GET",
-      `/api/public/datasets/dataset-name/runs/run-only-observation`,
+
+      `/api/public/datasets/${encodeURIComponent("dataset name")}/runs/${encodeURIComponent("run + only + observation")}`,
     );
     expect(getRunAPI.status).toBe(200);
     expect(getRunAPI.body).toMatchObject({
-      name: "run-only-observation",
+      name: "run + only + observation",
       description: "run-description",
       metadata: { key: "value" },
       datasetId: dataset.body.id,
-      datasetName: "dataset-name",
+      datasetName: "dataset name",
       datasetRunItems: expect.arrayContaining([
         expect.objectContaining({
           datasetItemId: "dataset-item-id",
           observationId: observationId,
           traceId: traceId,
-          datasetRunName: "run-only-observation",
+          datasetRunName: "run + only + observation",
         }),
       ]),
     });

web/src/__tests__/dom.clienttest.ts

@@ -0,0 +1,19 @@
+import * as DOMPurify from "dompurify";
+
+// Create test cases
+describe("should sanitize dom", () => {
+  it("should sanitize urls", () => {
+    const dirty =
+      "http://www.some.site/page.html?default=<script>alert(document.cookie)</script>";
+    const cleam = DOMPurify.sanitize(dirty);
+
+    expect(cleam).toBe("http://www.some.site/page.html?default=");
+  });
+  it("should should not sanitize valid langfuse urls", () => {
+    const validLangfuseUrl =
+      "https://cloud.langfuse.com/project/abjdnksadn/traces?filter=timestamp%3Bdatetime%3B%3B%3E%3B2024-04-19T00%253A00%253A00.000Z%2Cid%3Bstring%3B%3Bcontains%3Ba%2Ctimestamp%3Bdatetime%3B%3B%3C%3B2024-04-30T22%253A00%253A00.000Z%2CinputTokens%3Bnumber%3B%3B%3E%3B1";
+    const cleam = DOMPurify.sanitize(validLangfuseUrl);
+
+    expect(cleam).toBe(validLangfuseUrl);
+  });
+});

web/src/__tests__/ingestion.servertest.ts

@@ -281,6 +281,7 @@ describe("/api/public/ingestion API Endpoint", () => {
       expect(dbScore?.name).toBe("score-name");
       expect(dbScore?.value).toBe(100.5);
       expect(dbScore?.observationId).toBeNull();
+      expect(dbScore?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
     });
   });
 
@@ -816,57 +817,6 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbTrace.length).toBe(1);
   });
 
-  it("should fail for auth errors", async () => {
-    const traceId = v4();
-    const scoreId = v4();
-
-    const responseOne = await makeAPICall("POST", "/api/public/ingestion", {
-      batch: [
-        {
-          id: v4(),
-          type: "trace-create",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: traceId,
-            name: "trace-name",
-            userId: "user-1",
-            metadata: { key: "value" },
-            release: "1.0.0",
-            version: "2.0.0",
-          },
-        },
-        {
-          id: v4(),
-          type: "score-create",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: scoreId,
-            name: "score-name",
-            value: 100.5,
-            traceId: "some-random-id",
-          },
-        },
-      ],
-    });
-
-    expect(responseOne.status).toBe(207);
-
-    expect("errors" in responseOne.body).toBe(true);
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body.errors.length).toBe(1);
-    expect("successes" in responseOne.body).toBe(true);
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body.successes.length).toBe(1);
-
-    const dbTrace = await prisma.trace.findMany({
-      where: {
-        name: "trace-name",
-      },
-    });
-
-    expect(dbTrace.length).toBe(1);
-  });
-
   it("should fail for resource not found", async () => {
     const traceId = v4();
 
@@ -1653,121 +1603,6 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
     expect(dbScore?.value).toBe(scoreValue);
   });
 
-  it("should error on Bearer Auth for a trace from different project", async () => {
-    const otherProjectId = "other_project_id";
-    const traceId = "trace_id";
-    const bearerAuth = "Bearer pk-lf-1234567890";
-
-    const scoreId = "score_id";
-    const scoreEventId = "score_event_id";
-    const scoreName = "score-name";
-    const scoreValue = 100.5;
-
-    // Seed db with a trace to be scored
-    try {
-      await prisma.project.create({
-        data: {
-          id: otherProjectId,
-          name: "another-project",
-        },
-      });
-
-      await prisma.trace.create({
-        data: {
-          id: traceId,
-          name: "trace-name",
-          project: { connect: { id: otherProjectId } },
-        },
-      });
-
-      const response = await makeAPICall(
-        "POST",
-        "/api/public/ingestion",
-        {
-          batch: [
-            {
-              id: scoreEventId,
-              type: "score-create",
-              timestamp: new Date().toISOString(),
-              body: {
-                id: scoreId,
-                name: scoreName,
-                value: scoreValue,
-                traceId: traceId,
-              },
-            },
-          ],
-        },
-        bearerAuth,
-      );
-
-      expect(response.status).toBe(207);
-      expect(response.body.successes.length).toBe(0);
-      expect(response.body).toHaveProperty("errors");
-      expect(response.body.errors.length).toEqual(1);
-      expect(response.body.errors[0]?.id).toEqual(scoreEventId);
-
-      expect(await prisma.trace.count()).toBe(1);
-      expect(await prisma.trace.count({ where: { id: traceId } })).toBe(1);
-
-      const dbScore = await prisma.score.findUnique({
-        where: {
-          id: scoreId,
-        },
-      });
-
-      expect(dbScore).toBeNull();
-    } finally {
-      await prisma.project.delete({ where: { id: otherProjectId } });
-    }
-  });
-
-  it("should error on Bearer Auth with a trace that does not exist", async () => {
-    const traceId = "trace_id";
-    const bearerAuth = "Bearer pk-lf-1234567890";
-
-    const scoreId = "score_id";
-    const scoreEventId = "score_event_id";
-    const scoreName = "score-name";
-    const scoreValue = 100.5;
-
-    const response = await makeAPICall(
-      "POST",
-      "/api/public/ingestion",
-      {
-        batch: [
-          {
-            id: scoreEventId,
-            type: "score-create",
-            timestamp: new Date().toISOString(),
-            body: {
-              id: scoreId,
-              name: scoreName,
-              value: scoreValue,
-              traceId: traceId,
-            },
-          },
-        ],
-      },
-      bearerAuth,
-    );
-
-    expect(response.status).toBe(207);
-    expect(response.body.successes.length).toBe(0);
-    expect(response.body.errors.length).toBe(1);
-    expect(response.body.errors[0]?.id).toBe(scoreEventId);
-
-    expect(await prisma.trace.count()).toBe(0);
-
-    const dbScore = await prisma.score.findUnique({
-      where: {
-        id: scoreId,
-      },
-    });
-
-    expect(dbScore).toBeNull();
-  });
-
   it("should error on wrong input", async () => {
     const traceId = "trace_id";
     const bearerAuth = "Bearer pk-lf-1234567890";