chore: release v2.30.2

Bumps [react-responsive](https://github.com/yocontra/react-responsive) from 9.0.2 to 10.0.0.
- [Changelog](https://github.com/yocontra/react-responsive/blob/master/CHANGELOG.md)
- [Commits](https://github.com/yocontra/react-responsive/compare/v9.0.2...v10.0.0)

---
updated-dependencies:
- dependency-name: react-responsive
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.env.dev.example

@@ -37,4 +37,10 @@ ANTHROPIC_API_KEY=""
 
 # Set during docker build of application
 # Used to disable environment verification at build time
-# DOCKER_BUILD=1
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=mybasicauthsecret

.env.local.example

@@ -16,4 +16,11 @@ SALT="salt"
 
 # Prompt playground
 OPENAI_API_KEY=""
-ANTHROPIC_API_KEY=""
+ANTHROPIC_API_KEY=""
+
+# Redis
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+LANGFUSE_WORKER_PASSWORD=myworkerpassword

.env.prod.example

@@ -20,6 +20,9 @@ NEXTAUTH_URL="http://localhost:3000"
 NEXTAUTH_SECRET="secret"
 SALT="salt"
 
+# Use CSP headers to enforce HTTPS, optional
+LANGFUSE_CSP_ENFORCE_HTTPS="true"
+
 # Docker only, optional
 # PORT=3000
 # HOSTNAME=localhost
@@ -34,6 +37,7 @@ SALT="salt"
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
 # AUTH_DISABLE_USERNAME_PASSWORD=true
+# AUTH_DISABLE_SIGNUP=true
 
 # SSO, each group is optional
 # AUTH_GOOGLE_CLIENT_ID=
@@ -46,6 +50,14 @@ SALT="salt"
 # AUTH_AZURE_AD_CLIENT_SECRET=
 # AUTH_AZURE_AD_TENANT_ID=
 # AUTH_AZURE_ALLOW_ACCOUNT_LINKING=false
+# AUTH_OKTA_CLIENT_ID=
+# AUTH_OKTA_CLIENT_SECRET=
+# AUTH_OKTA_ISSUER=
+# AUTH_OKTA_ALLOW_ACCOUNT_LINKING=false
+# AUTH_AUTH0_CLIENT_ID=
+# AUTH_AUTH0_CLIENT_SECRET=
+# AUTH_AUTH0_ISSUER=
+# AUTH_AUTH0_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -65,11 +77,6 @@ SALT="salt"
 # DB_EXPORT_PAGE_SIZE=1000
 
 
-# Prompt playground
-# OPENAI_API_KEY=""
-# ANTHROPIC_API_KEY=""
-
-
 ### START Langfuse Cloud Config
 # Used for Langfuse Cloud deployments
 # Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
@@ -90,6 +97,14 @@ SALT="salt"
 # NEXT_SENTRY_PROJECT=
 # SENTRY_AUTH_TOKEN=
 
+# Cloudflare Turnstile
+# NEXT_PUBLIC_TURNSTILE_SITE_KEY=
+# TURNSTILE_SECRET_KEY=
+
+# Prompt playground
+# OPENAI_API_KEY=""
+# ANTHROPIC_API_KEY=""
+
 # Betterstack
 # LANGFUSE_TEAM_BETTERSTACK_TOKEN=
 
@@ -99,4 +114,10 @@ SALT="salt"
 # Crisp chat
 # NEXT_PUBLIC_CRISP_WEBSITE_ID=
 
+# Encryption
+# ENCRYPTION_KEY=
+
+# Admin API
+# ADMIN_API_KEY=
+
 ### END Langfuse Cloud Config

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Currently inactive
+# * @langfuse/maintainers

.github/dependabot.yml

@@ -6,37 +6,11 @@
 version: 2
 updates:
   - package-ecosystem: npm
-    directory: "/worker" # Location of package manifests
+    directory: "/" # Location of package manifests
+    rebase-strategy: "disabled" # use dependabot-rebase-stale
     schedule:
       interval: "daily"
-    rebase-strategy: "disabled" # use dependabot-rebase-stale
-    commit-message:
-      prefix: chore
-      prefix-development: chore
-      include: scope
-    ignore:
-      - dependency-name: "@types/node"
-      - dependency-name: "@trpc/*"
-    groups:
-      sentry:
-        patterns:
-          - "@sentry/*"
-      prisma:
-        patterns:
-          - "prisma"
-          - "@prisma/*"
-      next:
-        patterns:
-          - "eslint-config-next"
-          - "next"
-      patches:
-        update-types:
-          - "patch"
-  - package-ecosystem: npm
-    directory: "/web" # Location of package manifests
-    schedule:
-      interval: "daily"
-    rebase-strategy: "disabled" # use dependabot-rebase-stale
+    versioning-strategy: "increase"
     commit-message:
       prefix: chore
       prefix-development: chore

.github/workflows/ci.yml.template

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Start containers
-        run: docker-compose -f "docker-compose.yml" up -d --build
+        run: docker compose -f "docker-compose.yml" up -d --build
 
       - name: Setup pnpm
         uses: pnpm/action-setup@v2.2.4

.github/workflows/pipeline.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 8.15.5
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -54,11 +54,11 @@ jobs:
           file: ./worker/Dockerfile
           push: false
 
-  tests:
+  tests-web:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18, 20]
+        node-version: [20]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -68,7 +68,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 8.15.5
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -87,7 +87,7 @@ jobs:
 
       - name: Run + migrate
         run: |
-          docker-compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
 
       - name: Seed DB
@@ -102,7 +102,56 @@ jobs:
         run: (pnpm run start&)
 
       - name: run tests
-        run: pnpm run test
+        run: pnpm --filter=web run test
+
+  tests-worker:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20]
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+    steps:
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
+      - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 8.15.5
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+
+      - name: install dependencies
+        run: |
+          pnpm install
+
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          cp .env.dev.example web/.env
+
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
+
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm run db:seed
+
+      - name: Build
+        run: pnpm --filter=worker... run build
+
+      - name: run tests
+        run: pnpm --filter=worker run test
 
   e2e-tests:
     runs-on: ubuntu-latest
@@ -110,7 +159,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 8.15.5
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -128,7 +177,7 @@ jobs:
 
       - name: Run + migrate
         run: |
-          docker-compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
 
       - name: Seed DB
@@ -140,9 +189,7 @@ jobs:
         run: pnpm run build
 
       - name: Install playwright
-        run: |
-          npm add --global playwright
-          pnpm exec playwright install
+        run: pnpm --filter=web exec playwright install --with-deps
 
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e
@@ -150,7 +197,7 @@ jobs:
   all-ci-passed:
     # This allows us to have a branch protection rule for tests and deploys with matrix
     runs-on: ubuntu-latest
-    needs: [lint, tests, e2e-tests, test-docker-build]
+    needs: [lint, tests-web, tests-worker, e2e-tests, test-docker-build]
     if: always()
     steps:
       - name: Successful deploy
@@ -167,9 +214,6 @@ jobs:
     if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
     environment: "protected branches"
     runs-on: ubuntu-latest
-    env:
-      REGISTRY: ghcr.io
-      IMAGE_NAME: ${{ github.repository }}
     permissions:
       packages: write
       contents: read
@@ -177,7 +221,7 @@ jobs:
     steps:
       - uses: pnpm/action-setup@v3
         with:
-          version: 8
+          version: 8.15.5
       - name: Setup node
         uses: actions/setup-node@v3
         with:
@@ -187,19 +231,26 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Log in to the Container registry
+      - name: Log in to the GitHub Container registry
         uses: docker/login-action@v2
         with:
-          registry: ${{ env.REGISTRY }}
+          registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: Extract metadata (tags, labels) for Docker
-        id: meta
+        id: meta-web
         uses: docker/metadata-action@v4
         with:
           images: |
-            ${{ env.REGISTRY }}/${{ github.repository }}
+            ghcr.io/langfuse/langfuse # GitHub
+            langfuse/langfuse # Docker Hub
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -208,11 +259,35 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
 
-      - name: Build and push Docker image
+      - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:
           context: .
           file: ./web/Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-web.outputs.tags }}
+          labels: ${{ steps.meta-web.outputs.labels }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta-worker
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ghcr.io/langfuse/langfuse-worker # GitHub
+            langfuse/langfuse-worker # Docker Hub
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+
+      - name: Build and push Docker image (worker)
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./worker/Dockerfile
+          push: true
+          tags: ${{ steps.meta-worker.outputs.tags }}
+          labels: ${{ steps.meta-worker.outputs.labels }}

CONTRIBUTING.md

@@ -1,3 +1,5 @@
+![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
+
 # Contributing to Langfuse
 
 First off, thanks for taking the time to contribute! ❤️
@@ -55,8 +57,10 @@ A good first step is to search for open [issues](https://github.com/langfuse/lan
 flowchart TB
    subgraph s4["Clients"]
       subgraph s2["langfuse/langfuse-python"]
-         Python["Python SDK"]
+         Python["Python low-level SDK"]
+         Decorator["observe() decorator"] -->|extends| Python
          OAI["OpenAI drop-in replacement"] -->|extends| Python
+         Llamaindex["LlamaIndex Integration"] -->|extends| Python
          LCPYTHON["Langchain Python Integration"] -->|extends| Python
          Langflow -->|uses| LCPYTHON
          LiteLLM -->|uses| Python
@@ -116,9 +120,11 @@ We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https:/
 
 - `web`: is the main application package providing Frontend and Backend APIs for Langfuse.
 - `worker` (no production yet): contains an application for asynchronous processing of tasks. This package is not yet used in production.
-- `shared`: contains shared code between the above packages.
-- `config-eslint`: contains eslint configurations which are shared between the above packages.
-- `config-typescript`: contains typescript configurations which are shared between the above packages.
+- `packages`:
+  - `shared`: contains shared code between the above packages.
+  - `config-eslint`: contains eslint configurations which are shared between the above packages.
+  - `config-typescript`: contains typescript configurations which are shared between the above packages.
+- `ee`: contains all enterprise features. See [EE README](ee/README.md) for more details.
 
 ## Development Setup
 
@@ -153,11 +159,11 @@ Requirements
    All database migrations and configs are in the `shared` package.
 
    ```bash
-   pnpm run db:migrate
+   pnpm --filter=shared run db:migrate
 
    # Optional: seed the database
-   # pnpm --filter=shared run db:seed
-   # pnpm --filter=shared run db:seed:examples
+   # pnpm run db:seed
+   # pnpm run db:seed:examples
    # pnpm --filter=shared run db:seed:load
    ```
 
@@ -179,17 +185,39 @@ Requirements
 
   ```bash
   pnpm install
+  pnpm run dev
   pnpm --filter=web run dev # execute command only in one package
   ```
 
   In the root `package.json`, you can find scripts which are executed with turbo e.g. `turbo run dev`. These scripts are executed with the help of Turbo. Turbo executes the commands in all packages taking care of the correct order of execution. Task definitions can be found in the `turbo.config.js` file.
 
+- Run migrations
+
+  To run migrations, you can execute the following command.
+
+  ```bash
+  pnpm run db:migrate -- --name <name of the migration>
+  ```
+
 > [!NOTE]
 > If you frequently switch branches, use `pnpm run dx` instead of `pnpm run dev`. This command will install dependencies, reset the database (wipe and apply all migrations), and run the database seeder with example data before starting the development server.
 
 > [!NOTE]
 > If you find yourself stuck and want to clean the repo, execute `pnpm run nuke`. It will remove all node_modules and build files.
 
+## System behavior
+
+### Ingestion API `(/public/api/ingestion)`
+
+- the ingestion API takes different event types (creation and updates of traces, generations, spans, events)
+- The API loops through each event and:
+  - validates the event
+  - stores the event raw in the events table
+  - calculates tokens for `generations`
+  - matches models from the `models` table to model for `generations` events
+  - upserts the event in the `traces` or `observations` table
+- returns a `207` HTTP status code with a list of errors if any event failed to be ingested
+
 ## Commit messages
 
 On the main branch, we adhere to the best practices of [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/). All pull requests and branches are squash-merged to maintain a clean and readable history. This approach ensures the addition of a conventional commit message when merging contributions.

LICENSE

@@ -2,9 +2,9 @@ Copyright (c) 2023 Finto Technologies GmbH
 
 Portions of this software are licensed as follows:
 
-* All content that resides under the "ee/" directory of this repository, if that directory exists, is licensed under the license defined in "ee/LICENSE".
-* All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
-* Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
+- All content that resides under the "ee/" and/or "web/src/ee" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
+- Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -22,4 +22,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

README.md

@@ -1,25 +1,19 @@
+![Langfuse GitHub Banner](https://github.com/langfuse/langfuse/assets/121163007/6035f0f3-d691-4963-b5d0-10cf506e9d42)
+
 <div align="center">
-   <a href="https://langfuse.com">
-      <h1>🪢 Langfuse</h1>
-   </a>
- <div>
-      <h3> <a href="https://cloud.langfuse.com">
-         <strong>Sign up</strong>
-       </a> · 
-      <a href="https://langfuse.com/docs/deployment/self-host">
-         <strong>Self Host Langfuse</strong>
-      </a> · 
-      <a href="https://langfuse.com/demo">
-         <strong>Demo Project (live data)</strong>
-      </a>
+   <div>
+      <h3>
+         <a href="https://cloud.langfuse.com">
+            <strong>Sign up</strong>
+         </a> · 
+         <a href="https://langfuse.com/docs/deployment/self-host">
+            <strong>Self Host</strong>
+         </a> · 
+         <a href="https://langfuse.com/demo">
+            <strong>Demo (live data)</strong>
+         </a>
       </h3>
-   <h3>
-      Langfuse is the open source LLM engineering platform.   
-   </h3>
-    <div>
-     Debug, analyze and iterate - together
    </div>
-   </br>
    <div>
       <a href="https://langfuse.com/docs"><strong>Docs</strong></a> ·
       <a href="https://langfuse.com/issues"><strong>Report Bug</strong></a> ·
@@ -28,7 +22,10 @@
       <a href="https://langfuse.com/roadmap"><strong>Roadmap</strong></a> ·
       <a href="https://langfuse.com/discord"><strong>Discord</strong></a> 
    </div>
-   </br>
+   <span>Langfuse uses <a href="https://github.com/orgs/langfuse/discussions"><strong>Github Discussions</strong></a>  for Support and Feature Requests.</span>
+   <br/>
+   <br/>
+   <div>
       <img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License">
       <a href="https://www.ycombinator.com/companies/langfuse"><img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23"></a>
       <a href="https://github.com/langfuse/langfuse/pkgs/container/langfuse"><img alt="Docker Image" src="https://img.shields.io/badge/docker-langfuse-blue?logo=Docker&logoColor=white&style=flat-square"></a>
@@ -57,15 +54,9 @@
 ### Test
 
 - **Experiments:** Track and test app behaviour before deploying a new version
-  - Datasets let you test expected in and output pairs and benchmark performance before deployiong ([Datasets](https://langfuse.com/docs/datasets))
+  - Datasets let you test expected in and output pairs and benchmark performance before deploying ([Datasets](https://langfuse.com/docs/datasets))
   - Track versions and releases in your application ([Experimentation](https://langfuse.com/docs/experimentation), [Prompt Management](https://langfuse.com/docs/prompts))
 
-### Video: Langfuse in two minutes
-
-https://github.com/langfuse/langfuse/assets/2834609/6041347a-b517-4a11-8737-93ef8f8af49f
-
-_Muted by default, enable sound for voice-over_
-
 ## Get started
 
 ### Langfuse Cloud
@@ -98,25 +89,25 @@ Templated deployments: [Railway, GCP Cloud Run, AWS Fargate, Kubernetes and othe
 
 ### API Keys
 
-You require a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
+You need a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
 
 ### Ingesting Data · Instrumenting Your Application
 
-Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
+Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python (Decorators)](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
 
 You may want to use another integration to get started quickly or implement a use case that we do not yet support. However, we recommend to migrate to the Langfuse SDKs over time to ensure performance and stability.
 
-See our the [→ Quickstart](https://langfuse.com/docs/get-started) to get started in integrating Langfuse.
+See the [→ Quickstart](https://langfuse.com/docs/get-started) to integrate Langfuse.
 
 ### Integrations
 
-| Integration                                                      | Supports      | Description                                                      |
-| ---------------------------------------------------------------- | ------------- | ---------------------------------------------------------------- |
-| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility.      |
-| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python        | Automated instrumentation via drop-in replacement of OpenAI SDK. |
-| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.        |
-| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python       | Automated instrumentation via LlamaIndex callback system.        |
-| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.            |
+| Integration                                                      | Supports      | Description                                                 |
+| ---------------------------------------------------------------- | ------------- | ----------------------------------------------------------- |
+| [**SDK** - _recommended_](https://langfuse.com/docs/sdk)         | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
+| [OpenAI SDK](https://langfuse.com/docs/integrations/openai)      | Python, JS/TS | Automated instrumentation of OpenAI SDK.                    |
+| [Langchain](https://langfuse.com/docs/integrations/langchain)    | Python, JS/TS | Instrumentation via Langchain callbacks.                    |
+| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python        | Automated instrumentation via LlamaIndex callback system.   |
+| [API](https://langfuse.com/docs/api)                             |               | Directly call the public API. OpenAPI spec available.       |
 
 External projects/packages that integrate with Langfuse:
 
@@ -140,8 +131,8 @@ External projects/packages that integrate with Langfuse:
 In order of preference the best way to communicate with us:
 
 - [GitHub Discussions](https://github.com/orgs/langfuse/discussions): Contribute [ideas](https://langfuse.com/idea) [support requests](https://github.com/orgs/langfuse/discussions/categories/support) and [report bugs](https://github.com/langfuse/langfuse/issues/new?labels=%F0%9F%90%9E%E2%9D%94+unconfirmed+bug&projects=&template=bug_report.yml&title=bug%3A+) (preferred as we create a permanent, indexed artifact for other community members)
-- [Discord](https://langfuse.com/discord): For community support and to chat directly with maintainers
-- Privately: Email contact at langfuse dot com
+- [Discord](https://langfuse.com/discord): community support
+- Privately: contact at langfuse dot com
 
 ## Contributing to Langfuse
 
@@ -151,7 +142,7 @@ In order of preference the best way to communicate with us:
 
 ## License
 
-This repository is MIT licensed, except for the `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.
+This repository is MIT licensed, except for the `ee` folders. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.
 
 ## Misc
 

docker-compose.build.yml

@@ -16,11 +16,9 @@ services:
       - SALT=mysalt
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - REDIS_URL=${REDIS_URL:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_HOST=${LANGFUSE_WORKER_HOST:-worker}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
 
   worker:
@@ -30,7 +28,6 @@ services:
     depends_on:
       - db
       - redis
-      - server
     ports:
       - "3030:3030"
     environment:
@@ -39,13 +36,14 @@ services:
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
       - PORT=${PORT:-3030}
-      - REDIS_URL=${REDIS_URL:-redis}
+      - REDIS_HOST=${REDIS_HOST:-redis}
       - REDIS_PORT=${REDIS_PORT:-6379}
       - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      - LANGFUSE_WORKER_PASSWORD=${LANGFUSE_WORKER_PASSWORD:-mybasicauthsecret}
     restart: always
 
   redis:
-    image: redis
+    image: redis:7.2.4
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
     restart: always

docker-compose.dev.yml

@@ -2,7 +2,7 @@ version: "3.5"
 
 services:
   redis:
-    image: redis
+    image: redis:7.2.4
     restart: always
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}

docker-compose.yml

@@ -13,7 +13,6 @@ services:
       - SALT=mysalt
       - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
 
   db:

ee/.eslintrc.js

@@ -0,0 +1,8 @@
+/** @type {import("eslint").Linter.Config} */
+module.exports = {
+  extends: ["@repo/eslint-config/library.js"],
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    project: true,
+  },
+};

ee/Readme.md

@@ -1,3 +1,5 @@
 # Enterprise Edition
 
-While we are currently working 100% on the open source version, we consider financing the project by developing some premium features as part of an enterprise version under a commercial license. Please reach out to us if you have specific requirements: enterprise@langfuse.com
+This folder includes features that are only available in the Enterprise Edition of Langfuse and on Langfuse Cloud.
+
+See [LICENSE](../LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.

ee/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@langfuse/ee",
+  "version": "1.0.0",
+  "private": true,
+  "main": "./dist/src/index.js",
+  "types": "./dist/src/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    },
+    "./sso": {
+      "import": "./dist/src/sso/index.js",
+      "require": "./dist/src/sso/index.js"
+    }
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix"
+  },
+  "dependencies": {
+    "@langfuse/shared": "workspace:*",
+    "next": "^14.2.1",
+    "next-auth": "^4.24.7",
+    "zod": "^3.22.4"
+  },
+  "devDependencies": {
+    "@repo/eslint-config": "*",
+    "@repo/typescript-config": "*",
+    "@types/node": "^20.11.29",
+    "@typescript-eslint/parser": "^7.7.0",
+    "eslint": "^8.57.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-config-standard": "^17.1.0",
+    "eslint-plugin-prettier": "^5.1.3",
+    "prettier": "^3.2.5",
+    "ts-node": "^10.9.2",
+    "tsc-watch": "^6.2.0",
+    "typescript": "^5.4.5"
+  }
+}

ee/prettierrc.json

@@ -0,0 +1,4 @@
+{
+  "trailingComma": "es5",
+  "printWidth": 120
+}

ee/src/ee-license-check/index.ts

@@ -0,0 +1,4 @@
+import { env } from "../env";
+
+export const isEeAvailable: boolean =
+  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined;

ee/src/env.ts

@@ -0,0 +1,9 @@
+import { z } from "zod";
+import { env as sharedEnv } from "@langfuse/shared";
+
+const EnvSchema = z.object({
+  NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
+  ADMIN_API_KEY: z.string().optional(),
+});
+
+export const env = { ...sharedEnv, ...EnvSchema.parse(process.env) };

ee/src/index.ts

@@ -0,0 +1 @@
+export { isEeAvailable } from "./ee-license-check";

ee/src/sso/index.ts

@@ -0,0 +1,236 @@
+import { type Provider } from "next-auth/providers/index";
+import GoogleProvider from "next-auth/providers/google";
+import GitHubProvider from "next-auth/providers/github";
+import OktaProvider from "next-auth/providers/okta";
+import Auth0Provider from "next-auth/providers/auth0";
+import AzureADProvider from "next-auth/providers/azure-ad";
+import { isEeAvailable } from "..";
+import { prisma } from "@langfuse/shared/src/db";
+import { encrypt, decrypt } from "@langfuse/shared/encryption";
+import { SsoProviderSchema } from "./types";
+import { type NextApiRequest, type NextApiResponse } from "next";
+import { env } from "../env";
+
+// Local cache for SSO configurations
+let cachedSsoConfigs: {
+  data: SsoProviderSchema[];
+  timestamp: number;
+} | null = null;
+
+/**
+ * Get all SSO configurations from the database or from local cache and parse them into SsoProviderSchema objects.
+ *
+ * @returns {Promise<SsoProviderSchema[]>} - A list of all SSO configurations. Empty array if none are configured or EE is not available.
+ */
+async function getSsoConfigs(): Promise<SsoProviderSchema[]> {
+  if (!isEeAvailable) return [];
+  const CACHE_TTL = 60 * 1000; // 1 minute
+
+  // Set/refresh the cache if it's empty or expired
+  if (
+    cachedSsoConfigs === null ||
+    Date.now() - cachedSsoConfigs.timestamp > CACHE_TTL
+  ) {
+    // transform into zod object
+    const dbConfigs = await prisma.ssoConfig.findMany();
+    const parsedSsoConfigs = dbConfigs
+      .map((v) => {
+        try {
+          const parsedValue = SsoProviderSchema.parse(v);
+          return parsedValue;
+        } catch (e) {
+          console.error(
+            `Failed to parse SSO provider config for domain ${v.domain}`,
+            e
+          );
+          return null;
+        }
+      })
+      .filter((parsed) => parsed !== null) as SsoProviderSchema[];
+
+    cachedSsoConfigs = {
+      data: parsedSsoConfigs,
+      timestamp: Date.now(),
+    };
+  }
+
+  return cachedSsoConfigs.data;
+}
+
+/**
+ * Load all custom SSO providers from the database. To be used within `providers` in NextAuth backend configuration.
+ *
+ * @returns {Promise<Provider[]>} - A list of all custom SSO providers.
+ */
+export async function loadSsoProviders(): Promise<Provider[]> {
+  if (!isEeAvailable) return [];
+
+  const ssoConfigs = await getSsoConfigs();
+
+  const providers: Provider[] = [];
+
+  for (const dbSsoConfig of ssoConfigs) {
+    const provider = dbToNextAuthProvider(dbSsoConfig);
+    if (provider !== null) providers.push(provider);
+  }
+
+  return providers;
+}
+
+/**
+ * @returns `true` if any custom SSO provider is configured in the database.
+ */
+export async function isAnySsoConfigured(): Promise<boolean> {
+  if (!isEeAvailable) return false;
+  const ssoConfigs = await getSsoConfigs();
+  return ssoConfigs.length > 0;
+}
+
+/**
+ * Get the custom SSO providerId for a domain. To be used with NextAuth's `signIn(providerId)`.
+ *
+ * @param domain - The domain to check for a custom SSO provider, e.g. "example.com".
+ * @returns `providerId` or null if none is configured or EE is not available.
+ */
+export async function getSsoAuthProviderIdForDomain(
+  domain: string
+): Promise<string | null> {
+  if (!isEeAvailable) return null;
+  const ssoConfig = (await getSsoConfigs()).find(
+    (ssoConfig) => ssoConfig.domain === domain.toLowerCase()
+  );
+
+  if (!ssoConfig) return null;
+  return getAuthProviderIdForSsoConfig(ssoConfig);
+}
+
+/**
+ * Converts a SsoProviderConfig to a NextAuth Provider instance.
+ *
+ * @param {SsoProviderSchema} provider - The SSO configuration from the database.
+ * @returns {Provider | null} - A NextAuth Provider instance or null if parsing fails or no custom credentials are used for this SSO config.
+ */
+const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
+  // If the SsoConfig does not use custom credentials, return null as no additional provider needs to be added to NextAuth
+  if (!provider.authConfig) return null;
+
+  if (provider.authProvider === "google")
+    return GoogleProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "github")
+    return GitHubProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "auth0")
+    return Auth0Provider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "okta")
+    return OktaProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else if (provider.authProvider === "azure-ad")
+    return AzureADProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
+  else {
+    // Type check to ensure we handle all providers
+    // eslint-disable-next-line no-unused-vars
+    const _: never = provider;
+    throw new Error(
+      `Unrecognized SSO provider for domain ${(provider as any).domain}`
+    );
+  }
+};
+
+/**
+ * Get the custom SSO providerId for a database SSO configuration. To be used with NextAuth's `signIn(providerId)`.
+ *
+ * @param {DbSsoConfig} dbSsoConfig - The SSO configuration from the database.
+ * @returns {string} - The providerId used in NextAuth.
+ */
+const getAuthProviderIdForSsoConfig = (
+  dbSsoConfig: SsoProviderSchema
+): string => {
+  if (!dbSsoConfig.authConfig) return dbSsoConfig.authProvider;
+  return `${dbSsoConfig.domain}.${dbSsoConfig.authProvider}`;
+};
+
+export async function createNewSsoConfigHandler(
+  req: NextApiRequest,
+  res: NextApiResponse
+) {
+  try {
+    if (!isEeAvailable) {
+      res.status(403).json({ error: "EE is not available" });
+      return;
+    }
+    // allow only POST requests
+    if (req.method !== "POST") {
+      res.status(405).json({ error: "Method Not Allowed" });
+      return;
+    }
+    // check if ADMIN_API_KEY is set
+    if (!env.ADMIN_API_KEY) {
+      res.status(500).json({ error: "ADMIN_API_KEY is not set" });
+      return;
+    }
+    if (!env.ENCRYPTION_KEY) {
+      res.status(500).json({ error: "ENCRYPTION_KEY is not set" });
+      return;
+    }
+    // check bearer token
+    const { authorization } = req.headers;
+    if (!authorization) {
+      res
+        .status(401)
+        .json({ error: "Unauthorized: No authorization header provided" });
+      return;
+    }
+    const [scheme, token] = authorization.split(" ");
+    if (scheme !== "Bearer" || !token || token !== env.ADMIN_API_KEY) {
+      res.status(401).json({ error: "Unauthorized: Invalid token" });
+      return;
+    }
+
+    const body = SsoProviderSchema.safeParse(req.body);
+    if (!body.success) {
+      res.status(400).json({ error: body.error });
+      return;
+    }
+
+    const { domain, authProvider, authConfig } = body.data;
+
+    const encryptedClientSecret = authConfig
+      ? {
+          ...authConfig,
+          clientSecret: encrypt(authConfig.clientSecret),
+        }
+      : undefined;
+
+    await prisma.ssoConfig.create({
+      data: {
+        domain,
+        authProvider,
+        authConfig: encryptedClientSecret,
+      },
+    });
+    res.status(201).json({
+      message: "SSO configuration created successfully",
+    });
+  } catch (e) {
+    console.log(e);
+    res.status(500).json({ error: "Internal Server Error" });
+  }
+}

ee/src/sso/types.ts

@@ -0,0 +1,81 @@
+import { z } from "zod";
+
+const base = z.object({
+  domain: z.string().refine((v) => v === v.toLowerCase(), {
+    message: "Domain must be lowercase",
+  }),
+});
+
+export const GoogleProviderSchema = base.extend({
+  authProvider: z.literal("google"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const GithubProviderSchema = base.extend({
+  authProvider: z.literal("github"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const Auth0ProviderSchema = base.extend({
+  authProvider: z.literal("auth0"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const OktaProviderSchema = base.extend({
+  authProvider: z.literal("okta"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export const AzureAdProviderSchema = base.extend({
+  authProvider: z.literal("azure-ad"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      tenantId: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
+export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
+export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
+export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
+export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
+export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
+
+export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
+  GoogleProviderSchema,
+  GithubProviderSchema,
+  Auth0ProviderSchema,
+  OktaProviderSchema,
+  AzureAdProviderSchema,
+]);
+
+export type SsoProviderSchema = z.infer<typeof SsoProviderSchema>;

ee/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@repo/typescript-config/base.json",
+  "compilerOptions": {
+    "moduleResolution": "NodeNext",
+    "module": "NodeNext",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "types": ["node"],
+    "target": "ES2020",
+    "rootDir": "."
+  },
+  "include": ["."],
+  "exclude": ["node_modules", "dist"]
+}

fern/apis/server/definition/commons.yml

@@ -122,38 +122,61 @@ types:
       observationId: optional<string>
       timestamp: datetime
       comment: optional<string>
-  Dataset:
+  DatasetCore:
     properties:
       id: string
       name: string
+      description: optional<string>
+      metadata: optional<unknown>
       projectId: string
       createdAt: datetime
       updatedAt: datetime
+  DatasetWithReferences: # GET datasets
+    extends: DatasetCore
+    properties:
+      items:
+        type: list<string>
+        docs: list of dataset item ids
+      runs:
+        type: list<string>
+        docs: list of dataset run names
+  Dataset: # GET datasets/[name]
+    docs: Dataset including all items
+    extends: DatasetCore
+    properties:
       items: list<DatasetItem>
       runs: list<string>
   DatasetItem:
     properties:
       id: string
       status: DatasetStatus
-      input: unknown
+      input: optional<unknown>
       expectedOutput: optional<unknown>
+      metadata: optional<unknown>
+      sourceTraceId: optional<string>
       sourceObservationId: optional<string>
       datasetId: string
+      datasetName: string
       createdAt: datetime
       updatedAt: datetime
   DatasetRunItem:
     properties:
       id: string
       datasetRunId: string
+      datasetRunName: string
       datasetItemId: string
-      observationId: string
+      traceId: string
+      observationId: optional<string>
       createdAt: datetime
       updatedAt: datetime
   DatasetRun:
     properties:
       id: string
       name: string
+      description: optional<string>
+      metadata: optional<unknown>
       datasetId: string
+      datasetName: string
       createdAt: datetime
       updatedAt: datetime
       datasetRunItems: list<DatasetRunItem>
@@ -188,6 +211,7 @@ types:
     enum:
       - API
       - REVIEW
+      - EVAL
 
 errors:
   Error:

fern/apis/server/definition/dataset-items.yml

@@ -24,8 +24,11 @@ types:
   CreateDatasetItemRequest:
     properties:
       datasetName: string
-      input: unknown
+      input: optional<unknown>
       expectedOutput: optional<unknown>
+      metadata: optional<unknown>
+      sourceTraceId: optional<string>
+      sourceObservationId: optional<string>
       id:
         type: optional<string>
         docs: Dataset items are upserted on their id

fern/apis/server/definition/dataset-run-items.yml

@@ -15,5 +15,14 @@ types:
   CreateDatasetRunItemRequest:
     properties:
       runName: string
+      runDescription:
+        type: optional<string>
+        docs: Description of the run. If run exists, description will be updated.
+      metadata:
+        type: optional<unknown>
+        docs: Metadata of the dataset run, updates run if run already exists
       datasetItemId: string
-      observationId: string
+      observationId: optional<string>
+      traceId:
+        type: optional<string>
+        docs: traceId should always be provided. For compatibility with older SDK versions it can also be inferred from the provided observationId.

fern/apis/server/definition/datasets.yml

@@ -1,10 +1,21 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
 imports:
   commons: ./commons.yml
+  pagination: ./utils/pagination.yml
 service:
   auth: true
   base-path: /api/public
   endpoints:
+    list:
+      method: GET
+      docs: Get all datasets
+      path: /datasets
+      request:
+        name: GetDatasetsRequest
+        query-parameters:
+          page: optional<integer>
+          limit: optional<integer>
+      response: PaginatedDatasets
     get:
       method: GET
       docs: Get a dataset and its items
@@ -28,6 +39,12 @@ service:
       response: commons.DatasetRun
 
 types:
+  PaginatedDatasets:
+    properties:
+      data: list<commons.DatasetWithReferences>
+      meta: pagination.MetaResponse
   CreateDatasetRequest:
     properties:
       name: string
+      description: optional<string>
+      metadata: optional<unknown>

fern/apis/server/definition/metrics.yml

@@ -13,8 +13,12 @@ service:
       request:
         name: GetDailyMetricsRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
           traceName:
             type: optional<string>
             docs: Optional filter by the name of the trace
@@ -25,6 +29,12 @@ service:
             type: optional<string>
             allow-multiple: true
             docs: Optional filter for metrics where traces include all of these tags
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include traces on or after a certain timestamp
+          toTimestamp:
+            type: optional<datetime>
+            docs: Optional filter to only include traces before a certain timestamp
       response: DailyMetrics
 types:
   DailyMetrics:
@@ -37,12 +47,13 @@ types:
     properties:
       date: date
       countTraces: integer
+      countObservations: integer
       totalCost: double
       usage: list<UsageByModel>
   UsageByModel:
     docs: Daily usage of a given model. Usage corresponds to the unit set for the specific model (e.g. tokens).
     properties:
-      model: string
+      model: optional<string>
       inputUsage: integer
       outputUsage: integer
       totalUsage: integer

fern/apis/server/definition/observations.yml

@@ -22,13 +22,20 @@ service:
       request:
         name: GetObservationsRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1.
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           name: optional<string>
           userId: optional<string>
           type: optional<string>
           traceId: optional<string>
           parentObservationId: optional<string>
+          fromStartTime:
+            type: optional<datetime>
+            docs: Retrieve only observations with a start_time greater than this timestamp.
       response: ObservationsViews
 
 types:

fern/apis/server/definition/prompts.yml

@@ -25,6 +25,20 @@ service:
 
 types:
   CreatePromptRequest:
+    union:
+      chat: CreateChatPromptRequest
+      text: CreateTextPromptRequest
+
+  CreateChatPromptRequest:
+    properties:
+      name: string
+      isActive:
+        docs: Should the prompt be promoted to production immediately?
+        type: boolean
+      prompt: list<ChatMessage>
+      config: optional<unknown>
+
+  CreateTextPromptRequest:
     properties:
       name: string
       isActive:
@@ -34,8 +48,29 @@ types:
       config: optional<unknown>
 
   Prompt:
+    union:
+      chat: ChatPrompt
+      text: TextPrompt
+
+  BasePrompt:
     properties:
       name: string
       version: integer
-      prompt: string
       config: unknown
+
+  ChatMessage:
+    properties:
+      role:
+        type: string
+      content:
+        type: string
+
+  TextPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: string
+
+  ChatPrompt:
+    extends: BasePrompt
+    properties:
+      prompt: list<ChatMessage>

fern/apis/server/definition/score.yml

@@ -19,10 +19,26 @@ service:
       request:
         name: GetScoresRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1.
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           userId: optional<string>
           name: optional<string>
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Retrieve only scores newer than this timestamp.
+          source:
+            type: commons.ScoreSource
+            docs: Retrieve only scores from a specific source.
+          operator:
+            type: optional<string>
+            docs: Retrieve only scores with <operator> value.
+          value:
+            type: optional<double>
+            docs: Retrieve only scores with <operator> value.
       response: Scores
     get-by-id:
       docs: Get a score

fern/apis/server/definition/trace.yml

@@ -16,16 +16,23 @@ service:
           docs: The unique langfuse identifier of a trace
       response: commons.TraceWithFullDetails
     list:
-      docs: Get list of traces
+      docs: Get list of traces.
       method: GET
       path: /traces
       request:
         name: GetTracesRequest
         query-parameters:
-          page: optional<integer>
-          limit: optional<integer>
+          page:
+            type: optional<integer>
+            docs: Page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit.
           userId: optional<string>
           name: optional<string>
+          fromTimestamp:
+            type: optional<datetime>
+            docs: Retrieve only traces newer than this timestamp.
           orderBy:
             type: optional<string>
             docs: "Format of the string [field].[asc/desc]. Fields: id, timestamp, name, userId, release, version, public, bookmarked, sessionId. Example: timestamp.asc"

fern/apis/server/generators.yml

@@ -11,14 +11,16 @@ groups:
           namespaceExport: Langfuse
           allowCustomFetcher: true
       - name: fernapi/fern-python-sdk
-        version: 0.7.3
+        version: 1.1.0-rc2
         output:
           location: local-file-system
           path: ../../../generated/python
         config:
           client_class_name: FernLangfuse
+          improved_imports: false
           pydantic_config:
             require_optional_fields: false
+            use_str_enums: false
       - name: fernapi/fern-typescript-node-sdk
         version: 0.7.1
         output:

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "finto",
-  "version": "0.16.36"
+  "version": "0.21.0"
 }

generated/openapi-client/openapi.yml

@@ -1,104 +0,0 @@
-openapi: 3.0.1
-info:
-  title: langfuse
-  version: ''
-paths:
-  /api/public/scores:
-    post:
-      description: Add a score to the database, upserts on id
-      operationId: score_create
-      tags:
-        - Score
-      parameters: []
-      responses:
-        '200':
-          description: ''
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Score'
-        '400':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '401':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '403':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '405':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-      security:
-        - BearerAuth: []
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/CreateScoreRequest'
-components:
-  schemas:
-    CreateScoreRequest:
-      title: CreateScoreRequest
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        comment:
-          type: string
-      required:
-        - traceId
-        - name
-        - value
-    Score:
-      title: Score
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        timestamp:
-          type: string
-          format: date-time
-        comment:
-          type: string
-      required:
-        - id
-        - traceId
-        - name
-        - value
-        - timestamp
-  securitySchemes:
-    BearerAuth:
-      type: http
-      scheme: bearer

generated/postman/collection.json

@@ -65,7 +65,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"id\": \"example\"\n}",
+              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"metadata\": \"UNKNOWN\",\n    \"sourceTraceId\": \"example\",\n    \"sourceObservationId\": \"example\",\n    \"id\": \"example\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -137,7 +137,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"runName\": \"example\",\n    \"datasetItemId\": \"example\",\n    \"observationId\": \"example\"\n}",
+              "raw": "{\n    \"runName\": \"example\",\n    \"runDescription\": \"example\",\n    \"metadata\": \"UNKNOWN\",\n    \"datasetItemId\": \"example\",\n    \"observationId\": \"example\",\n    \"traceId\": \"example\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -154,6 +154,42 @@
       "description": null,
       "name": "Datasets",
       "item": [
+        {
+          "_type": "endpoint",
+          "name": "List",
+          "request": {
+            "description": "Get all datasets",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/datasets?page=&limit=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "datasets"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": null
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
         {
           "_type": "endpoint",
           "name": "Get",
@@ -209,7 +245,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"name\": \"example\"\n}",
+              "raw": "{\n    \"name\": \"example\",\n    \"description\": \"example\",\n    \"metadata\": \"UNKNOWN\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -343,7 +379,7 @@
           "request": {
             "description": "Get daily metrics of the Langfuse project",
             "url": {
-              "raw": "{{baseUrl}}/api/public/metrics/daily?page=&limit=&traceName=&userId=&tags=",
+              "raw": "{{baseUrl}}/api/public/metrics/daily?page=&limit=&traceName=&userId=&tags=&fromTimestamp=&toTimestamp=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -357,12 +393,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "limit of items per page"
                 },
                 {
                   "key": "traceName",
@@ -378,6 +414,16 @@
                   "key": "tags",
                   "value": "",
                   "description": "Optional filter for metrics where traces include all of these tags"
+                },
+                {
+                  "key": "fromTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include traces on or after a certain timestamp"
+                },
+                {
+                  "key": "toTimestamp",
+                  "value": "",
+                  "description": "Optional filter to only include traces before a certain timestamp"
                 }
               ],
               "variable": []
@@ -434,7 +480,7 @@
           "request": {
             "description": "Get a list of observations",
             "url": {
-              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&parentObservationId=",
+              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&parentObservationId=&fromStartTime=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -447,12 +493,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "Page number, starts at 1."
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
                 },
                 {
                   "key": "name",
@@ -478,6 +524,11 @@
                   "key": "parentObservationId",
                   "value": "",
                   "description": null
+                },
+                {
+                  "key": "fromStartTime",
+                  "value": "",
+                  "description": "Retrieve only observations with a start_time greater than this timestamp."
                 }
               ],
               "variable": []
@@ -587,7 +638,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": \"example\",\n    \"config\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -643,7 +694,7 @@
           "request": {
             "description": "Get a list of scores",
             "url": {
-              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=",
+              "raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=&fromTimestamp=&source=&operator=&value=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -656,12 +707,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "Page number, starts at 1."
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
                 },
                 {
                   "key": "userId",
@@ -672,6 +723,26 @@
                   "key": "name",
                   "value": "",
                   "description": null
+                },
+                {
+                  "key": "fromTimestamp",
+                  "value": "",
+                  "description": "Retrieve only scores newer than this timestamp."
+                },
+                {
+                  "key": "source",
+                  "value": "",
+                  "description": "Retrieve only scores from a specific source."
+                },
+                {
+                  "key": "operator",
+                  "value": "",
+                  "description": "Retrieve only scores with <operator> value."
+                },
+                {
+                  "key": "value",
+                  "value": "",
+                  "description": "Retrieve only scores with <operator> value."
                 }
               ],
               "variable": []
@@ -829,9 +900,9 @@
           "_type": "endpoint",
           "name": "List",
           "request": {
-            "description": "Get list of traces",
+            "description": "Get list of traces.",
             "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&orderBy=&tags=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&fromTimestamp=&orderBy=&tags=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -844,12 +915,12 @@
                 {
                   "key": "page",
                   "value": "",
-                  "description": null
+                  "description": "Page number, starts at 1"
                 },
                 {
                   "key": "limit",
                   "value": "",
-                  "description": null
+                  "description": "Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit."
                 },
                 {
                   "key": "userId",
@@ -861,6 +932,11 @@
                   "value": "",
                   "description": null
                 },
+                {
+                  "key": "fromTimestamp",
+                  "value": "",
+                  "description": "Retrieve only traces newer than this timestamp."
+                },
                 {
                   "key": "orderBy",
                   "value": "",

package.json

@@ -1,26 +1,73 @@
 {
   "name": "langfuse",
-  "version": "2.7.0",
+  "version": "2.30.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
+  "private": true,
+  "engines": {
+    "node": ">=20",
+    "pnpm": "<9"
+  },
   "scripts": {
     "preinstall": "npx only-allow pnpm",
-    "infra:dev:up": "docker-compose -f ./docker-compose.dev.yml up -d",
-    "infra:dev:down": "docker-compose -f ./docker-compose.dev.yml down",
+    "infra:dev:up": "docker compose -f ./docker-compose.dev.yml up -d",
+    "infra:dev:down": "docker compose -f ./docker-compose.dev.yml down",
     "db:generate": "turbo run db:generate",
     "db:migrate": "turbo run db:migrate",
     "db:seed": "turbo run db:seed",
+    "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=worker run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
     "lint": "turbo run lint",
-    "test": "turbo run test"
+    "test": "turbo run test",
+    "models:migrate": "turbo run models:migrate",
+    "release": "dotenv -e ../.env -- release-it"
   },
   "devDependencies": {
-    "dotenv-cli": "^7.3.0",
+    "@release-it/bumper": "^6.0.1",
+    "dotenv-cli": "^7.4.1",
     "prettier": "^3.2.5",
-    "turbo": "^1.13.0"
+    "release-it": "^17.2.0",
+    "turbo": "^1.13.2"
+  },
+  "release-it": {
+    "git": {
+      "commitMessage": "chore: release v${version}",
+      "tagName": "v${version}"
+    },
+    "plugins": {
+      "@release-it/bumper": {
+        "out": [
+          {
+            "file": "./web/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./worker/src/constants/VERSION.ts",
+            "type": "application/typescript"
+          },
+          {
+            "file": "./web/package.json"
+          },
+          {
+            "file": "./worker/package.json"
+          }
+        ]
+      }
+    },
+    "github": {
+      "release": true,
+      "web": true,
+      "autoGenerate": true,
+      "releaseName": "v${version}",
+      "comments": {
+        "submit": true,
+        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
+        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
+      }
+    }
   }
 }

packages/config-eslint/package.json

@@ -8,12 +8,13 @@
     "next.js"
   ],
   "devDependencies": {
-    "@vercel/style-guide": "^5.2.0",
-    "eslint-config-turbo": "^1.12.4",
-    "eslint-config-prettier": "^9.1.0",
-    "eslint-plugin-only-warn": "^1.1.0",
-    "@typescript-eslint/parser": "^7.1.0",
     "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "typescript": "^5.3.3"
+    "@typescript-eslint/parser": "^7.7.0",
+    "@vercel/style-guide": "^6.0.0",
+    "eslint-config-next": "^14.2.1",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-config-turbo": "^1.13.2",
+    "eslint-plugin-only-warn": "^1.1.0",
+    "typescript": "^5.4.5"
   }
 }

packages/config-typescript/base.json

@@ -10,6 +10,7 @@
     "inlineSources": false,
     "isolatedModules": true,
     "module": "ESNext",
+    "noErrorTruncation": true,
     "moduleResolution": "Bundler",
     "noUnusedLocals": false,
     "noUnusedParameters": false,

packages/shared/package.json

@@ -5,18 +5,34 @@
   "private": true,
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
-  "files": [
-    "dist/**"
-  ],
   "engines": {
     "node": ">=20"
   },
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    },
+    "./src/db": {
+      "import": "./dist/src/db.js",
+      "require": "./dist/src/db.js"
+    },
+    "./src/server/auth": {
+      "import": "./dist/src/server/auth.js",
+      "require": "./dist/src/server/auth.js"
+    },
+    "./encryption": {
+      "import": "./dist/src/encryption/index.js",
+      "require": "./dist/src/encryption/index.js"
+    }
+  },
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
     "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
     "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
+    "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
     "db:reset": "dotenv -e ../../.env npx -- prisma migrate reset",
     "db:deploy": "dotenv -e ../../.env npx -- prisma migrate deploy",
     "db:seed": "dotenv -e ../../.env -- npx prisma db seed",
@@ -28,26 +44,26 @@
     "seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
-    "@langchain/anthropic": "^0.1.8",
-    "@langchain/core": "^0.1.49",
-    "@langchain/openai": "^0.0.23",
-    "@prisma/client": "^5.11.0",
+    "@langchain/anthropic": "^0.1.14",
+    "@langchain/core": "^0.1.58",
+    "@langchain/openai": "^0.0.28",
+    "@prisma/client": "^5.12.1",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.28",
+    "langchain": "^0.1.33",
     "prisma-extension-kysely": "^2.1.0",
-    "prisma-kysely": "^1.8.0",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "zod-to-json-schema": "^3.22.5"
   },
   "devDependencies": {
     "@repo/eslint-config": "*",
     "@repo/typescript-config": "*",
     "@types/lodash": "^4.14.202",
     "@types/node": "^20.11.29",
-    "@types/pg": "^8.11.2",
+    "@types/pg": "^8.11.5",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/parser": "^7.2.0",
+    "@typescript-eslint/parser": "^7.7.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
@@ -55,12 +71,12 @@
     "kysely-codegen": "^0.11.0",
     "nodemon": "^3.0.3",
     "prettier": "^3.2.5",
-    "prisma": "^5.11.0",
+    "prisma": "^5.12.1",
     "prisma-erd-generator": "^1.11.2",
+    "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
-    "tsc-watch": "^6.0.4",
-    "tsup": "^8.0.2",
-    "typescript": "^5.3.3",
-    "vitest": "^1.3.1"
+    "tsc-watch": "^6.2.0",
+    "typescript": "^5.4.5",
+    "vitest": "^1.5.0"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -26,7 +26,8 @@ export const ObservationLevel = {
 export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
 export const ScoreSource = {
     API: "API",
-    REVIEW: "REVIEW"
+    REVIEW: "REVIEW",
+    EVAL: "EVAL"
 } as const;
 export type ScoreSource = (typeof ScoreSource)[keyof typeof ScoreSource];
 export const PricingUnit = {
@@ -45,6 +46,22 @@ export const DatasetStatus = {
     ARCHIVED: "ARCHIVED"
 } as const;
 export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
+export const JobType = {
+    EVAL: "EVAL"
+} as const;
+export type JobType = (typeof JobType)[keyof typeof JobType];
+export const JobConfigState = {
+    ACTIVE: "ACTIVE",
+    INACTIVE: "INACTIVE"
+} as const;
+export type JobConfigState = (typeof JobConfigState)[keyof typeof JobConfigState];
+export const JobExecutionStatus = {
+    COMPLETED: "COMPLETED",
+    ERROR: "ERROR",
+    PENDING: "PENDING",
+    CANCELLED: "CANCELLED"
+} as const;
+export type JobExecutionStatus = (typeof JobExecutionStatus)[keyof typeof JobExecutionStatus];
 export type Account = {
     id: string;
     user_id: string;
@@ -95,6 +112,8 @@ export type CronJobs = {
 export type Dataset = {
     id: string;
     name: string;
+    description: string | null;
+    metadata: unknown | null;
     project_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
@@ -102,8 +121,10 @@ export type Dataset = {
 export type DatasetItem = {
     id: string;
     status: Generated<DatasetStatus>;
-    input: unknown;
+    input: unknown | null;
     expected_output: unknown | null;
+    metadata: unknown | null;
+    source_trace_id: string | null;
     source_observation_id: string | null;
     dataset_id: string;
     created_at: Generated<Timestamp>;
@@ -113,17 +134,33 @@ export type DatasetRunItems = {
     id: string;
     dataset_run_id: string;
     dataset_item_id: string;
-    observation_id: string;
+    trace_id: string;
+    observation_id: string | null;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
 export type DatasetRuns = {
     id: string;
     name: string;
+    description: string | null;
+    metadata: unknown | null;
     dataset_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
+export type EvalTemplate = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    name: string;
+    version: number;
+    prompt: string;
+    model: string;
+    model_params: unknown;
+    vars: Generated<string[]>;
+    output_schema: unknown;
+};
 export type Events = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -134,10 +171,33 @@ export type Events = {
     url: string | null;
     method: string | null;
 };
-export type Example = {
+export type JobConfiguration = {
     id: string;
     created_at: Generated<Timestamp>;
-    updated_at: Timestamp;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_type: JobType;
+    status: Generated<JobConfigState>;
+    eval_template_id: string | null;
+    score_name: string;
+    filter: unknown;
+    target_object: string;
+    variable_mapping: unknown;
+    sampling: string;
+    delay: number;
+};
+export type JobExecution = {
+    id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    project_id: string;
+    job_configuration_id: string;
+    status: JobExecutionStatus;
+    start_time: Timestamp | null;
+    end_time: Timestamp | null;
+    error: string | null;
+    job_input_trace_id: string | null;
+    job_output_score_id: string | null;
 };
 export type Membership = {
     project_id: string;
@@ -232,6 +292,13 @@ export type ObservationView = {
     calculated_total_cost: string | null;
     latency: number | null;
 };
+export type PosthogIntegration = {
+    project_id: string;
+    encrypted_posthog_api_key: string;
+    posthog_host_name: string;
+    last_sync_at: Timestamp | null;
+    enabled: boolean;
+};
 export type Pricing = {
     id: string;
     model_name: string;
@@ -253,11 +320,13 @@ export type Prompt = {
     updated_at: Generated<Timestamp>;
     project_id: string;
     created_by: string;
-    prompt: string;
+    prompt: unknown;
     name: string;
     version: number;
+    type: Generated<string>;
     is_active: boolean;
     config: Generated<unknown>;
+    tags: Generated<string[]>;
 };
 export type Score = {
     id: string;
@@ -275,6 +344,13 @@ export type Session = {
     user_id: string;
     expires: Timestamp;
 };
+export type SsoConfig = {
+    domain: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    auth_provider: string;
+    auth_config: unknown | null;
+};
 export type Trace = {
     id: string;
     external_id: string | null;
@@ -300,6 +376,24 @@ export type TraceSession = {
     bookmarked: Generated<boolean>;
     public: Generated<boolean>;
 };
+export type TraceView = {
+    id: string;
+    external_id: string | null;
+    timestamp: Generated<Timestamp>;
+    name: string | null;
+    user_id: string | null;
+    metadata: unknown | null;
+    release: string | null;
+    version: string | null;
+    project_id: string;
+    public: Generated<boolean>;
+    bookmarked: Generated<boolean>;
+    tags: Generated<string[]>;
+    input: unknown | null;
+    output: unknown | null;
+    session_id: string | null;
+    duration: number | null;
+};
 export type User = {
     id: string;
     name: string | null;
@@ -326,20 +420,25 @@ export type DB = {
     dataset_run_items: DatasetRunItems;
     dataset_runs: DatasetRuns;
     datasets: Dataset;
+    eval_templates: EvalTemplate;
     events: Events;
-    Example: Example;
+    job_configurations: JobConfiguration;
+    job_executions: JobExecution;
     membership_invitations: MembershipInvitation;
     memberships: Membership;
     models: Model;
     observations: Observation;
     observations_view: ObservationView;
+    posthog_integrations: PosthogIntegration;
     pricings: Pricing;
     projects: Project;
     prompts: Prompt;
     scores: Score;
     Session: Session;
+    sso_configs: SsoConfig;
     trace_sessions: TraceSession;
     traces: Trace;
+    traces_view: TraceView;
     users: User;
     verification_tokens: VerificationToken;
 };

packages/shared/prisma/migrations/20240325211959_remove_example_table/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the `Example` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropTable
+DROP TABLE "Example";

packages/shared/prisma/migrations/20240325212245_dataset_runs_add_metadata/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "dataset_runs" ADD COLUMN     "metadata" JSONB;

packages/shared/prisma/migrations/20240326114211_dataset_run_item_bind_to_trace/migration.sql

@@ -0,0 +1,6 @@
+-- AlterTable
+ALTER TABLE "dataset_run_items" ADD COLUMN     "trace_id" TEXT,
+ALTER COLUMN "observation_id" DROP NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "dataset_run_items" ADD CONSTRAINT "dataset_run_items_trace_id_fkey" FOREIGN KEY ("trace_id") REFERENCES "traces"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240326114337_dataset_run_item_backfill_trace_id/migration.sql

@@ -0,0 +1,5 @@
+-- Backfill trace_id for existing run_items based on the linked observation
+UPDATE dataset_run_items
+SET trace_id = observations.trace_id
+FROM observations 
+WHERE dataset_run_items.observation_id = observations.id;

packages/shared/prisma/migrations/20240326115136_dataset_run_item_traceid_non_null/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Made the column `trace_id` on table `dataset_run_items` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "dataset_run_items" ALTER COLUMN "trace_id" SET NOT NULL;

packages/shared/prisma/migrations/20240326115424_dataset_run_item_index_trace/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "dataset_run_items_trace_id_idx" ON "dataset_run_items"("trace_id");

packages/shared/prisma/migrations/20240328065738_dataset_item_input_nullable/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "dataset_items" ALTER COLUMN "input" DROP NOT NULL;

packages/shared/prisma/migrations/20240404203640_dataset_item_source_trace_id/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "dataset_items" ADD COLUMN     "source_trace_id" TEXT;
+
+-- AddForeignKey
+ALTER TABLE "dataset_items" ADD CONSTRAINT "dataset_items_source_trace_id_fkey" FOREIGN KEY ("source_trace_id") REFERENCES "traces"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240404210315_dataset_add_descriptions/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "dataset_runs" ADD COLUMN     "description" TEXT;
+
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "description" TEXT;

packages/shared/prisma/migrations/20240404232317_dataset_items_backfill_source_trace_id/migration.sql

@@ -0,0 +1,7 @@
+-- Backfill source_trace_id for existing dataset_items based on the linked source_observation_id
+UPDATE dataset_items
+SET source_trace_id = observations.trace_id
+FROM observations
+WHERE dataset_items.source_observation_id = observations.id
+  AND dataset_items.source_observation_id IS NOT NULL
+  AND dataset_items.source_trace_id IS NULL

packages/shared/prisma/migrations/20240405124810_prompt_to_json/migration.sql

@@ -0,0 +1,21 @@
+BEGIN;
+
+ALTER TABLE prompts
+ADD COLUMN json_prompt JSONB;
+
+UPDATE prompts
+SET json_prompt = to_json(prompt::text)::json;
+
+ALTER TABLE prompts
+DROP COLUMN prompt;
+
+ALTER TABLE prompts
+RENAME COLUMN json_prompt TO prompt;
+
+ALTER TABLE prompts
+ALTER COLUMN prompt SET NOT NULL;
+
+ALTER TABLE prompts
+ADD COLUMN type TEXT NOT NULL DEFAULT 'text';
+
+COMMIT;

packages/shared/prisma/migrations/20240408133037_add_objects_for_evals/migration.sql

@@ -0,0 +1,102 @@
+-- CreateEnum
+CREATE TYPE "JobType" AS ENUM ('EVAL');
+
+-- CreateEnum
+CREATE TYPE "JobExecutionStatus" AS ENUM ('COMPLETED', 'ERROR', 'PENDING', 'CANCELLED');
+
+-- AlterEnum
+ALTER TYPE "ScoreSource" ADD VALUE 'EVAL';
+
+-- CreateTable
+CREATE TABLE "eval_templates" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "version" INTEGER NOT NULL,
+    "prompt" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "model_params" JSONB NOT NULL,
+    "vars" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "output_schema" JSONB NOT NULL,
+
+    CONSTRAINT "eval_templates_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_configurations" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_type" "JobType" NOT NULL,
+    "eval_template_id" TEXT,
+    "score_name" TEXT NOT NULL,
+    "filter" JSONB NOT NULL,
+    "target_object" TEXT NOT NULL,
+    "variable_mapping" JSONB NOT NULL,
+    "sampling" DECIMAL(65,30) NOT NULL,
+    "delay" INTEGER NOT NULL,
+
+    CONSTRAINT "job_configurations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "job_executions" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "job_configuration_id" TEXT NOT NULL,
+    "status" "JobExecutionStatus" NOT NULL,
+    "start_time" TIMESTAMP(3),
+    "end_time" TIMESTAMP(3),
+    "error" TEXT,
+    "job_input_trace_id" TEXT,
+    "job_output_score_id" TEXT,
+
+    CONSTRAINT "job_executions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_id_idx" ON "eval_templates"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "eval_templates_project_id_idx" ON "eval_templates"("project_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "eval_templates_project_id_name_version_key" ON "eval_templates"("project_id", "name", "version");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_id_idx" ON "job_configurations"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_configurations_project_id_idx" ON "job_configurations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_id_idx" ON "job_executions"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "job_executions_project_id_idx" ON "job_executions"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "eval_templates" ADD CONSTRAINT "eval_templates_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_configurations" ADD CONSTRAINT "job_configurations_eval_template_id_fkey" FOREIGN KEY ("eval_template_id") REFERENCES "eval_templates"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_configuration_id_fkey" FOREIGN KEY ("job_configuration_id") REFERENCES "job_configurations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_input_trace_id_fkey" FOREIGN KEY ("job_input_trace_id") REFERENCES "traces"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "job_executions" ADD CONSTRAINT "job_executions_job_output_score_id_fkey" FOREIGN KEY ("job_output_score_id") REFERENCES "scores"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/shared/prisma/migrations/20240408134328_prompt_table_add_tags/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "prompts" ADD COLUMN     "tags" TEXT[] DEFAULT ARRAY[]::TEXT[];

packages/shared/prisma/migrations/20240408134330_prompt_table_add_index_to_tags/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "prompts_tags_idx" ON "prompts" USING GIN ("tags" array_ops);

packages/shared/prisma/migrations/20240411134330_model_updates/migration.sql

@@ -0,0 +1,30 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://ai.google.dev/models/gemini#model-variations
+  -- https://cloud.google.com/vertex-ai/generative-ai/pricing
+  ('cluv2sjeo000008ih0fv23hi0', NULL, 'gemini-1.0-pro-latest', '(?i)^(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  -- stable versions priced differently
+  -- https://developers.googleblog.com/2024/02/gemini-15-available-for-private-preview-in-google-ai-studio.html
+  ('cluv2subq000108ih2mlrga6a', NULL, 'gemini-1.0-pro', '(?i)^(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2sx04000208ihbek75lsz', NULL, 'gemini-1.0-pro-001', '(?i)^(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+  ('cluv2szw0000308ihch3n79x7', NULL, 'gemini-pro', '(?i)^(gemini-pro)(@[a-zA-Z0-9]+)?$', '2024-02-15', 0.000000125, 0.000000375, NULL, 'CHARACTERS', NULL, NULL),
+
+  ('cluv2t2x0000408ihfytl45l1', NULL, 'gemini-1.5-pro-latest', '(?i)^(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$', NULL, 0.0000025, 0.0000075, NULL, 'CHARACTERS', NULL, NULL),
+
+  -- https://platform.openai.com/docs/models/continuous-model-upgrades
+  ('cluv2t5k3000508ih5kve9zag', NULL, 'gpt-4-turbo-2024-04-09', '(?i)^(gpt-4-turbo-2024-04-09)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }')
+
+
+  

packages/shared/prisma/migrations/20240411194142_update_job_config_status/migration.sql

@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "JobConfigState" AS ENUM ('ACTIVE', 'INACTIVE');
+
+-- AlterTable
+ALTER TABLE "job_configurations" ADD COLUMN     "status" "JobConfigState" NOT NULL DEFAULT 'ACTIVE';

packages/shared/prisma/migrations/20240411224142_update_models/migration.sql

@@ -0,0 +1,16 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://platform.openai.com/docs/models/continuous-model-upgrades
+  ('cluvpl4ls000008l6h2gx3i07', NULL, 'gpt-4-turbo', '(?i)^(gpt-4-turbo)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }')

packages/shared/prisma/migrations/20240414203636_ee_add_sso_configs/migration.sql

@@ -0,0 +1,10 @@
+-- CreateTable
+CREATE TABLE "sso_configs" (
+    "domain" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "auth_provider" TEXT NOT NULL,
+    "auth_config" JSONB,
+
+    CONSTRAINT "sso_configs_pkey" PRIMARY KEY ("domain")
+);

packages/shared/prisma/migrations/20240415235737_index_models_model_name/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "models_model_name_idx" ON "models"("model_name");

packages/shared/prisma/migrations/20240416173813_add_internal_model_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_internal_model_idx" ON "observations"("internal_model");

packages/shared/prisma/migrations/20240417102742_metadata_on_dataset_and_dataset_item/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "dataset_items" ADD COLUMN     "metadata" JSONB;
+
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "metadata" JSONB;

packages/shared/prisma/migrations/20240419152924_posthog_integration_settings/migration.sql

@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE "posthog_integrations" (
+    "project_id" TEXT NOT NULL,
+    "encrypted_posthog_api_key" TEXT NOT NULL,
+    "posthog_host_name" TEXT NOT NULL,
+    "last_sync_at" TIMESTAMP(3),
+    "enabled" BOOLEAN NOT NULL,
+
+    CONSTRAINT "posthog_integrations_pkey" PRIMARY KEY ("project_id")
+);
+
+-- CreateIndex
+CREATE INDEX "posthog_integrations_project_id_idx" ON "posthog_integrations"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "posthog_integrations" ADD CONSTRAINT "posthog_integrations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/schema.prisma

@@ -32,12 +32,6 @@ generator kysely {
     // enumFileName = "enums.ts"
 }
 
-model Example {
-    id        String   @id @default(cuid())
-    createdAt DateTime @default(now()) @map("created_at")
-    updatedAt DateTime @updatedAt @map("updated_at")
-}
-
 // Necessary for Next auth
 model Account {
     id                String  @id @default(cuid())
@@ -98,22 +92,26 @@ model VerificationToken {
 }
 
 model Project {
-    id           String                 @id @default(cuid())
-    createdAt    DateTime               @default(now()) @map("created_at")
-    updatedAt    DateTime               @default(now()) @updatedAt @map("updated_at")
-    name         String
-    cloudConfig  Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
-    members      Membership[]
-    traces       Trace[]
-    observations Observation[]
-    apiKeys      ApiKey[]
-    dataset      Dataset[]
-    RawEvents    Events[]
-    invitations  MembershipInvitation[]
-    sessions     TraceSession[]
-    Prompt       Prompt[]
-    Model        Model[]
-    AuditLog     AuditLog[]
+    id                 String                 @id @default(cuid())
+    createdAt          DateTime               @default(now()) @map("created_at")
+    updatedAt          DateTime               @default(now()) @updatedAt @map("updated_at")
+    name               String
+    cloudConfig        Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
+    members            Membership[]
+    traces             Trace[]
+    observations       Observation[]
+    apiKeys            ApiKey[]
+    dataset            Dataset[]
+    RawEvents          Events[]
+    invitations        MembershipInvitation[]
+    sessions           TraceSession[]
+    Prompt             Prompt[]
+    Model              Model[]
+    AuditLog           AuditLog[]
+    EvalTemplate       EvalTemplate[]
+    JobConfiguration   JobConfiguration[]
+    JobExecution       JobExecution[]
+    PosthogIntegration PosthogIntegration[]
 
     @@map("projects")
 }
@@ -191,6 +189,8 @@ model TraceSession {
     @@map("trace_sessions")
 }
 
+// Update TraceView below when making changes to this model!
+
 model Trace {
     id         String        @id @default(cuid())
     externalId String?       @map("external_id")
@@ -210,7 +210,10 @@ model Trace {
     sessionId  String?       @map("session_id")
     session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
 
-    scores Score[]
+    scores          Score[]
+    DatasetRunItems DatasetRunItems[]
+    DatasetItem     DatasetItem[]
+    JobExecution    JobExecution[]
 
     @@index([projectId])
     @@index([sessionId])
@@ -224,6 +227,33 @@ model Trace {
     @@map("traces")
 }
 
+// This view is based on the trace table. Once prisma supports
+// inheritance, we should remove code duplication here.
+view TraceView {
+    // trace fields
+    id         String   @id @default(cuid())
+    externalId String?  @map("external_id")
+    timestamp  DateTime @default(now())
+    name       String?
+    userId     String?  @map("user_id")
+    metadata   Json?
+    release    String?
+    version    String?
+    projectId  String   @map("project_id")
+    public     Boolean  @default(false)
+    bookmarked Boolean  @default(false)
+    tags       String[] @default([])
+    input      Json?
+    output     Json?
+    sessionId  String?  @map("session_id")
+
+    // calculated fields
+    duration Float? @map("duration") // can be null if no observations in trace
+
+    @@map("traces_view")
+}
+
+// Update ObservationView below when making changes to this model!
 model Observation {
     id                  String           @id @default(cuid())
     traceId             String?          @map("trace_id")
@@ -273,214 +303,12 @@ model Observation {
     @@index(projectId)
     @@index(parentObservationId)
     @@index(model)
+    @@index(internalModel)
     @@index(promptId)
     @@index([projectId, startTime, type])
     @@map("observations")
 }
 
-enum ObservationType {
-    SPAN
-    EVENT
-    GENERATION
-}
-
-enum ObservationLevel {
-    DEBUG
-    DEFAULT
-    WARNING
-    ERROR
-}
-
-model Score {
-    id            String       @id @default(cuid())
-    timestamp     DateTime     @default(now())
-    name          String
-    value         Float
-    source        ScoreSource
-    comment       String?
-    traceId       String       @map("trace_id")
-    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-    observationId String?      @map("observation_id")
-    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: SetNull)
-
-    @@unique([id, traceId]) // used for upsert
-    @@index(timestamp)
-    @@index([value])
-    @@index([traceId], type: Hash)
-    @@index([observationId], type: Hash)
-    @@index([source])
-    @@map("scores")
-}
-
-enum ScoreSource {
-    API
-    REVIEW
-}
-
-enum PricingUnit {
-    PER_1000_TOKENS
-    PER_1000_CHARS
-}
-
-enum TokenType {
-    PROMPT
-    COMPLETION
-    TOTAL
-}
-
-model Pricing {
-    id          String      @id @default(cuid())
-    modelName   String      @map("model_name")
-    pricingUnit PricingUnit @default(PER_1000_TOKENS) @map("pricing_unit")
-    price       Decimal
-    currency    String      @default("USD")
-    tokenType   TokenType   @map("token_type")
-
-    @@index(modelName)
-    @@map("pricings")
-}
-
-model CronJobs {
-    name         String    @id
-    lastRun      DateTime? @map("last_run")
-    jobStartedAt DateTime? @map("job_started_at")
-    state        String?
-
-    @@map("cron_jobs")
-}
-
-model Dataset {
-    id           String        @id @default(cuid())
-    name         String
-    projectId    String        @map("project_id")
-    project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    createdAt    DateTime      @default(now()) @map("created_at")
-    updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
-    datasetItems DatasetItem[]
-    datasetRuns  DatasetRuns[]
-
-    @@unique([projectId, name])
-    @@index([projectId], type: Hash)
-    @@map("datasets")
-}
-
-model DatasetItem {
-    id                  String            @id @default(cuid())
-    status              DatasetStatus     @default(ACTIVE)
-    input               Json
-    expectedOutput      Json?             @map("expected_output")
-    sourceObservationId String?           @map("source_observation_id")
-    sourceObservation   Observation?      @relation(fields: [sourceObservationId], references: [id], onDelete: SetNull)
-    datasetId           String            @map("dataset_id")
-    dataset             Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
-    createdAt           DateTime          @default(now()) @map("created_at")
-    updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
-    datasetRunItems     DatasetRunItems[]
-
-    @@index([sourceObservationId], type: Hash)
-    @@index([datasetId], type: Hash)
-    @@map("dataset_items")
-}
-
-enum DatasetStatus {
-    ACTIVE
-    ARCHIVED
-}
-
-model DatasetRuns {
-    id              String            @id @default(cuid())
-    name            String
-    datasetId       String            @map("dataset_id")
-    dataset         Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
-    createdAt       DateTime          @default(now()) @map("created_at")
-    updatedAt       DateTime          @default(now()) @updatedAt @map("updated_at")
-    datasetRunItems DatasetRunItems[]
-
-    @@unique([datasetId, name])
-    @@index([datasetId], type: Hash)
-    @@map("dataset_runs")
-}
-
-model DatasetRunItems {
-    id            String      @id @default(cuid())
-    datasetRunId  String      @map("dataset_run_id")
-    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
-    datasetItemId String      @map("dataset_item_id")
-    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
-    observationId String      @map("observation_id")
-    observation   Observation @relation(fields: [observationId], references: [id], onDelete: Cascade)
-    createdAt     DateTime    @default(now()) @map("created_at")
-    updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
-
-    @@index([datasetRunId], type: Hash)
-    @@index([datasetItemId], type: Hash)
-    @@index([observationId], type: Hash)
-    @@map("dataset_run_items")
-}
-
-model Events {
-    id        String   @id @default(cuid())
-    createdAt DateTime @default(now()) @map("created_at")
-    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
-    projectId String   @map("project_id")
-    project   Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
-    data      Json
-    headers   Json     @default("{}")
-    url       String?
-    method    String?
-
-    @@index(projectId)
-    @@map("events")
-}
-
-model Prompt {
-    id        String   @id @default(cuid())
-    createdAt DateTime @default(now()) @map("created_at")
-    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
-
-    projectId String  @map("project_id")
-    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
-
-    createdBy String @map("created_by")
-
-    prompt      String
-    name        String
-    version     Int
-    isActive    Boolean       @map("is_active")
-    config      Json          @default("{}")
-    Observation Observation[]
-
-    @@unique([projectId, name, version])
-    @@index([projectId, name, version])
-    @@index([projectId, id])
-    @@index([projectId])
-    @@map("prompts")
-}
-
-model Model {
-    id        String   @id @default(cuid())
-    createdAt DateTime @default(now()) @map("created_at")
-    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
-
-    projectId String?  @map("project_id")
-    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
-
-    modelName       String    @map("model_name")
-    matchPattern    String    @map("match_pattern")
-    startDate       DateTime? @map("start_date")
-    inputPrice      Decimal?  @map("input_price")
-    outputPrice     Decimal?  @map("output_price")
-    totalPrice      Decimal?  @map("total_price")
-    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, or IMAGES
-    tokenizerId     String?   @map("tokenizer_id")
-    tokenizerConfig Json?     @map("tokenizer_config")
-
-    @@unique([projectId, modelName, startDate, unit])
-    @@index([projectId, modelName, startDate, unit])
-    @@index([projectId, modelName])
-    @@map("models")
-}
-
 // This view is a mix of the observation and model. Once prisma supports
 // inheritance, we should remove code duplication here.
 view ObservationView {
@@ -526,6 +354,226 @@ view ObservationView {
     @@map("observations_view")
 }
 
+enum ObservationType {
+    SPAN
+    EVENT
+    GENERATION
+}
+
+enum ObservationLevel {
+    DEBUG
+    DEFAULT
+    WARNING
+    ERROR
+}
+
+model Score {
+    id            String         @id @default(cuid())
+    timestamp     DateTime       @default(now())
+    name          String
+    value         Float
+    source        ScoreSource
+    comment       String?
+    traceId       String         @map("trace_id")
+    trace         Trace          @relation(fields: [traceId], references: [id], onDelete: Cascade)
+    observationId String?        @map("observation_id")
+    observation   Observation?   @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    JobExecution  JobExecution[]
+
+    @@unique([id, traceId]) // used for upsert
+    @@index(timestamp)
+    @@index([value])
+    @@index([traceId], type: Hash)
+    @@index([observationId], type: Hash)
+    @@index([source])
+    @@map("scores")
+}
+
+enum ScoreSource {
+    API
+    REVIEW
+    EVAL
+}
+
+enum PricingUnit {
+    PER_1000_TOKENS
+    PER_1000_CHARS
+}
+
+enum TokenType {
+    PROMPT
+    COMPLETION
+    TOTAL
+}
+
+model Pricing {
+    id          String      @id @default(cuid())
+    modelName   String      @map("model_name")
+    pricingUnit PricingUnit @default(PER_1000_TOKENS) @map("pricing_unit")
+    price       Decimal
+    currency    String      @default("USD")
+    tokenType   TokenType   @map("token_type")
+
+    @@index(modelName)
+    @@map("pricings")
+}
+
+model CronJobs {
+    name         String    @id
+    lastRun      DateTime? @map("last_run")
+    jobStartedAt DateTime? @map("job_started_at")
+    state        String?
+
+    @@map("cron_jobs")
+}
+
+model Dataset {
+    id           String        @id @default(cuid())
+    name         String
+    description  String?
+    metadata     Json?
+    projectId    String        @map("project_id")
+    project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    createdAt    DateTime      @default(now()) @map("created_at")
+    updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
+    datasetItems DatasetItem[]
+    datasetRuns  DatasetRuns[]
+
+    @@unique([projectId, name])
+    @@index([projectId], type: Hash)
+    @@map("datasets")
+}
+
+model DatasetItem {
+    id                  String            @id @default(cuid())
+    status              DatasetStatus     @default(ACTIVE)
+    input               Json?
+    expectedOutput      Json?             @map("expected_output")
+    metadata            Json?
+    sourceTraceId       String?           @map("source_trace_id")
+    sourceTrace         Trace?            @relation(fields: [sourceTraceId], references: [id], onDelete: SetNull)
+    sourceObservationId String?           @map("source_observation_id")
+    sourceObservation   Observation?      @relation(fields: [sourceObservationId], references: [id], onDelete: SetNull)
+    datasetId           String            @map("dataset_id")
+    dataset             Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    createdAt           DateTime          @default(now()) @map("created_at")
+    updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
+    datasetRunItems     DatasetRunItems[]
+
+    @@index([sourceObservationId], type: Hash)
+    @@index([datasetId], type: Hash)
+    @@map("dataset_items")
+}
+
+enum DatasetStatus {
+    ACTIVE
+    ARCHIVED
+}
+
+model DatasetRuns {
+    id              String            @id @default(cuid())
+    name            String
+    description     String?
+    metadata        Json?
+    datasetId       String            @map("dataset_id")
+    dataset         Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    createdAt       DateTime          @default(now()) @map("created_at")
+    updatedAt       DateTime          @default(now()) @updatedAt @map("updated_at")
+    datasetRunItems DatasetRunItems[]
+
+    @@unique([datasetId, name])
+    @@index([datasetId], type: Hash)
+    @@map("dataset_runs")
+}
+
+model DatasetRunItems {
+    id            String       @id @default(cuid())
+    datasetRunId  String       @map("dataset_run_id")
+    datasetRun    DatasetRuns  @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetItemId String       @map("dataset_item_id")
+    datasetItem   DatasetItem  @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    traceId       String       @map("trace_id")
+    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
+    observationId String?      @map("observation_id")
+    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: Cascade)
+    createdAt     DateTime     @default(now()) @map("created_at")
+    updatedAt     DateTime     @default(now()) @updatedAt @map("updated_at")
+
+    @@index([datasetRunId], type: Hash)
+    @@index([datasetItemId], type: Hash)
+    @@index([observationId], type: Hash)
+    @@index([traceId])
+    @@map("dataset_run_items")
+}
+
+model Events {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+    projectId String   @map("project_id")
+    project   Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    data      Json
+    headers   Json     @default("{}")
+    url       String?
+    method    String?
+
+    @@index(projectId)
+    @@map("events")
+}
+
+model Prompt {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    createdBy String @map("created_by")
+
+    prompt      Json
+    name        String
+    version     Int
+    type        String        @default("text")
+    isActive    Boolean       @map("is_active")
+    config      Json          @default("{}")
+    tags        String[]      @default([])
+    Observation Observation[]
+
+    @@unique([projectId, name, version])
+    @@index([projectId, name, version])
+    @@index([projectId, id])
+    @@index([projectId])
+    @@index([tags(ops: ArrayOps)], type: Gin)
+    @@map("prompts")
+}
+
+// Update ObservationView below when making changes to this model!
+model Model {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String?  @map("project_id")
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    modelName       String    @map("model_name")
+    matchPattern    String    @map("match_pattern")
+    startDate       DateTime? @map("start_date")
+    inputPrice      Decimal?  @map("input_price")
+    outputPrice     Decimal?  @map("output_price")
+    totalPrice      Decimal?  @map("total_price")
+    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, or IMAGES
+    tokenizerId     String?   @map("tokenizer_id")
+    tokenizerConfig Json?     @map("tokenizer_config")
+
+    @@unique([projectId, modelName, startDate, unit])
+    @@index([projectId, modelName, startDate, unit])
+    @@index([projectId, modelName])
+    @@index(modelName)
+    @@map("models")
+}
+
 model AuditLog {
     id              String         @id @default(cuid())
     createdAt       DateTime       @default(now()) @map("created_at")
@@ -545,3 +593,122 @@ model AuditLog {
     @@index([createdAt])
     @@map("audit_logs")
 }
+
+model EvalTemplate {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    name             String
+    version          Int
+    prompt           String
+    model            String
+    modelParams      Json               @map("model_params")
+    vars             String[]           @default([])
+    outputSchema     Json               @map("output_schema")
+    JobConfiguration JobConfiguration[]
+
+    @@unique([projectId, name, version])
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("eval_templates")
+}
+
+enum JobType {
+    EVAL
+}
+
+enum JobConfigState {
+    ACTIVE
+    INACTIVE
+}
+
+model JobConfiguration {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobType        JobType        @map("job_type")
+    status         JobConfigState @default(ACTIVE)
+    evalTemplateId String?        @map("eval_template_id")
+    evalTemplate   EvalTemplate?  @relation(fields: [evalTemplateId], references: [id], onDelete: SetNull)
+
+    scoreName       String         @map("score_name")
+    filter          Json
+    targetObject    String         @map("target_object")
+    variableMapping Json           @map("variable_mapping")
+    sampling        Decimal // ratio of jobs that are executed for sampling (0..1)  
+    delay           Int // delay in milliseconds
+    JobExecution    JobExecution[]
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_configurations")
+}
+
+enum JobExecutionStatus {
+    COMPLETED
+    ERROR
+    PENDING
+    CANCELLED
+}
+
+model JobExecution {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    jobConfigurationId String           @map("job_configuration_id")
+    jobConfiguration   JobConfiguration @relation(fields: [jobConfigurationId], references: [id], onDelete: Cascade)
+
+    status    JobExecutionStatus
+    startTime DateTime?          @map("start_time")
+    endTime   DateTime?          @map("end_time")
+    error     String?
+
+    jobInputTraceId String? @map("job_input_trace_id")
+    trace           Trace?  @relation(fields: [jobInputTraceId], references: [id], onDelete: SetNull) // job remains when traces are deleted
+
+    jobOutputScoreId String? @map("job_output_score_id")
+    score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
+
+    @@index([projectId, id])
+    @@index([projectId])
+    @@map("job_executions")
+}
+
+// Single Sign-On configuration for a domain
+// This feature is part of the Enterprise Edition
+model SsoConfig {
+    domain       String   @id @default(cuid()) // e.g. "google.com"
+    createdAt    DateTime @default(now()) @map("created_at")
+    updatedAt    DateTime @default(now()) @updatedAt @map("updated_at")
+    authProvider String   @map("auth_provider") // e.g. "okta", ee/sso/types.ts
+
+    authConfig Json? @map("auth_config")
+    // e.g. { "clientId": "1234", "clientSecret": "5678" }, null if credentials from env should be used
+    // secrets like clientSecret are encrypted on the application level
+
+    @@map("sso_configs")
+}
+
+model PosthogIntegration {
+    projectId              String    @id @map("project_id")
+    project                Project   @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    encryptedPosthogApiKey String    @map("encrypted_posthog_api_key")
+    posthogHostName        String    @map("posthog_host_name")
+    lastSyncAt             DateTime? @map("last_sync_at")
+    enabled                Boolean
+
+    @@index([projectId])
+    @@map("posthog_integrations")
+}

packages/shared/prisma/seed.ts

@@ -4,13 +4,14 @@ import {
   type Prisma,
   ObservationType,
   ScoreSource,
-} from "../src/db";
+} from "../src/index";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
 
 import { chunk } from "lodash";
 import { v4 } from "uuid";
-import { ModelUsageUnit, getDisplaySecretKey, hashSecretKey } from "../src";
+import { ModelUsageUnit } from "../src";
+import { getDisplaySecretKey, hashSecretKey } from "../src/server/auth";
 
 const LOAD_TRACE_VOLUME = 10_000;
 
@@ -149,15 +150,14 @@ async function main() {
 
     const traceVolume = environment === "load" ? LOAD_TRACE_VOLUME : 100;
 
-    const { generationIds, traces, observations, scores, sessions, events } =
-      createObjects(
-        traceVolume,
-        envTags,
-        colorTags,
-        project1,
-        project2,
-        promptIds
-      );
+    const { traces, observations, scores, sessions, events } = createObjects(
+      traceVolume,
+      envTags,
+      colorTags,
+      project1,
+      project2,
+      promptIds
+    );
 
     console.log(
       `Seeding ${traces.length} traces, ${observations.length} observations, and ${scores.length} scores`
@@ -165,49 +165,146 @@ async function main() {
 
     await uploadObjects(traces, observations, scores, sessions, events);
 
+    // add eval objects
+    const evalTemplate = await prisma.evalTemplate.upsert({
+      where: {
+        projectId_name_version: {
+          projectId: project1.id,
+          name: "toxicity-template",
+          version: 1,
+        },
+      },
+      create: {
+        projectId: project1.id,
+        name: "toxicity-template",
+        version: 1,
+        prompt:
+          "Please evaluate the toxicity of the following text {{input}} {{output}}",
+        model: "gpt-3.5-turbo",
+        vars: ["input", "output"],
+        outputSchema: {
+          score: "provide a score between 0 and 1",
+          reasoning: "one sentence reasoning for the score",
+        },
+        modelParams: {
+          temperature: 0.7,
+          outputTokenLimit: 100,
+          topP: 0.9,
+        },
+      },
+      update: {},
+    });
+
+    await prisma.jobConfiguration.upsert({
+      where: {
+        id: "toxicity-job",
+      },
+      create: {
+        id: "toxicity-job",
+        evalTemplateId: evalTemplate.id,
+        projectId: project1.id,
+        jobType: "EVAL",
+        status: "ACTIVE",
+        scoreName: "toxicity",
+        filter: [
+          {
+            type: "string",
+            value: "user",
+            column: "User ID",
+            operator: "contains",
+          },
+        ],
+        variableMapping: [
+          {
+            langfuseObject: "trace",
+            selectedColumnId: "input",
+            templateVariable: "input",
+          },
+          {
+            langfuseObject: "trace",
+            selectedColumnId: "metadata",
+            templateVariable: "output",
+          },
+        ],
+        targetObject: "trace",
+        sampling: 1,
+        delay: 5_000,
+      },
+      update: {},
+    });
+
     for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
       const dataset = await prisma.dataset.create({
         data: {
           name: `demo-dataset-${datasetNumber}`,
+          description:
+            datasetNumber === 0 ? "Dataset test description" : undefined,
           projectId: project2.id,
+          metadata: datasetNumber === 0 ? { key: "value" } : undefined,
         },
       });
 
-      for (let datasetRunNumber = 0; datasetRunNumber < 2; datasetRunNumber++) {
+      const datasetItemIds = [];
+      for (let i = 0; i < 18; i++) {
+        const sourceObservation =
+          Math.random() > 0.3
+            ? observations[Math.floor(Math.random() * observations.length)]
+            : undefined;
+        const datasetItem = await prisma.datasetItem.create({
+          data: {
+            datasetId: dataset.id,
+            sourceTraceId: sourceObservation?.traceId,
+            sourceObservationId:
+              Math.random() > 0.5 ? sourceObservation?.id : undefined,
+            input:
+              Math.random() > 0.3
+                ? [
+                    {
+                      role: "user",
+                      content: "How can i create a React component?",
+                    },
+                  ]
+                : undefined,
+            expectedOutput:
+              Math.random() > 0.3
+                ? "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both."
+                : undefined,
+            metadata: Math.random() > 0.5 ? { key: "value" } : undefined,
+          },
+        });
+        datasetItemIds.push(datasetItem.id);
+      }
+
+      for (let datasetRunNumber = 0; datasetRunNumber < 5; datasetRunNumber++) {
         const datasetRun = await prisma.datasetRuns.create({
           data: {
             name: `demo-dataset-run-${datasetRunNumber}`,
+            description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,
+            metadata: [
+              undefined,
+              "string",
+              100,
+              { key: "value" },
+              ["tag1", "tag2"],
+            ][datasetRunNumber % 5],
           },
         });
 
-        for (let runNumber = 0; runNumber < 10; runNumber++) {
-          //pick randomly from existingSpanIds
-          const sourceObservationId =
-            generationIds[Math.floor(Math.random() * generationIds.length)];
-          const runObservationId =
-            generationIds[Math.floor(Math.random() * generationIds.length)];
-
-          const datasetItem = await prisma.datasetItem.create({
-            data: {
-              datasetId: dataset.id,
-              sourceObservationId:
-                Math.random() > 0.5 ? sourceObservationId : undefined,
-              input: [
-                {
-                  role: "user",
-                  content: "How can i create a React component?",
-                },
-              ],
-              expectedOutput:
-                "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.",
-            },
-          });
+        for (const datasetItemId of datasetItemIds) {
+          const relevantObservations = observations.filter(
+            (o) => o.projectId === project2.id
+          );
+          const observation =
+            relevantObservations[
+              Math.floor(Math.random() * relevantObservations.length)
+            ];
 
           await prisma.datasetRunItems.create({
             data: {
-              datasetItemId: datasetItem.id,
-              observationId: runObservationId!,
+              datasetItemId,
+              traceId: observation.traceId as string,
+              observationId: Math.random() > 0.5 ? observation.id : undefined,
               datasetRunId: datasetRun.id,
             },
           });
@@ -343,7 +440,6 @@ function createObjects(
   const observations: Prisma.ObservationCreateManyInput[] = [];
   const scores: Prisma.ScoreCreateManyInput[] = [];
   const sessions: Prisma.TraceSessionCreateManyInput[] = [];
-  const generationIds: string[] = [];
   const events: Prisma.ObservationCreateManyInput[] = [];
 
   for (let i = 0; i < traceVolume; i++) {
@@ -384,7 +480,7 @@ function createObjects(
         more: "1,2,3;4?6",
       },
       tags: tags as string[],
-      userId: `user-${i % 60}`,
+      userId: Math.random() > 0.3 ? `user-${i % 60}` : undefined,
       input:
         Math.random() > 0.3 ? "I'm looking for a React component" : undefined,
       output:
@@ -607,8 +703,6 @@ function createObjects(
             source: ScoreSource.API,
           });
 
-        generationIds.push(generation.id);
-
         for (let l = 0; l < Math.floor(Math.random() * 2); l++) {
           // random start time within span
           const eventTs = new Date(
@@ -640,7 +734,6 @@ function createObjects(
   ).map((session) => JSON.parse(session) as Prisma.TraceSessionCreateManyInput);
 
   return {
-    generationIds,
     traces,
     observations,
     scores,
@@ -691,6 +784,16 @@ async function generatePrompts(project: Project) {
       version: 1,
       isActive: true,
     },
+    {
+      id: `prompt-${v4()}`,
+      projectId: project.id,
+      createdBy: "user-1",
+      prompt: "Prompt 4 content",
+      name: "Prompt 4",
+      version: 1,
+      isActive: true,
+      tags: ["tag1", "tag2"],
+    },
   ];
 
   for (const prompt of prompts) {
@@ -710,6 +813,7 @@ async function generatePrompts(project: Project) {
         name: prompt.name,
         version: prompt.version,
         isActive: prompt.isActive,
+        tags: prompt.tags,
       },
       update: {
         id: prompt.id,

packages/shared/src/db.ts

@@ -3,6 +3,14 @@
 
 import { PrismaClient } from "@prisma/client";
 import { env } from "process";
+import kyselyExtension from "prisma-extension-kysely";
+import {
+  Kysely,
+  PostgresAdapter,
+  PostgresIntrospector,
+  PostgresQueryCompiler,
+} from "kysely";
+import { DB } from ".";
 
 // Instantiated according to the Prisma documentation
 // https://www.prisma.io/docs/orm/more/help-and-troubleshooting/help-articles/nextjs-prisma-client-dev-practices
@@ -16,12 +24,32 @@ const prismaClientSingleton = () => {
   });
 };
 
+const kyselySingleton = (prismaClient: PrismaClient) => {
+  return prismaClient.$extends(
+    kyselyExtension({
+      kysely: (driver) =>
+        new Kysely<DB>({
+          dialect: {
+            // This is where the magic happens!
+            createDriver: () => driver,
+            // Don't forget to customize these to match your database!
+            createAdapter: () => new PostgresAdapter(),
+            createIntrospector: (db) => new PostgresIntrospector(db),
+            createQueryCompiler: () => new PostgresQueryCompiler(),
+          },
+        }),
+    })
+  );
+};
 declare global {
   // eslint-disable-next-line no-var
   var prisma: undefined | ReturnType<typeof prismaClientSingleton>;
+  var kyselyPrisma: undefined | ReturnType<typeof kyselySingleton>;
 }
 
 export const prisma = globalThis.prisma ?? prismaClientSingleton();
+export const kyselyPrisma = globalThis.kyselyPrisma ?? kyselySingleton(prisma);
+
 export * from "@prisma/client";
 
 if (process.env.NODE_ENV !== "production") globalThis.prisma = prisma;

packages/shared/src/encryption/index.ts

@@ -0,0 +1,58 @@
+import crypto from "crypto";
+import { env } from "../env";
+
+const ENCRYPTION_KEY: string = env.ENCRYPTION_KEY || ""; // Must be 256 bits (32 bytes, 64 hex characters)
+const IV_LENGTH: number = 16; // For AES, this is always 16
+
+// Alternatively: openssl rand -hex 32
+export function keyGen() {
+  return crypto.randomBytes(32).toString("hex");
+}
+
+/**
+ * Encrypts the given plain text using AES-256-GCM algorithm.
+ *
+ * @param {string} plainText - The text to encrypt.
+ * @param {string} [keyHex=ENCRYPTION_KEY] - The encryption key in hexadecimal format.
+ * @returns {string} The encrypted data in hex format, including IV and authentication tag.
+ */
+export function encrypt(
+  plainText: string,
+  keyHex: string = ENCRYPTION_KEY
+): string {
+  const iv = crypto.randomBytes(IV_LENGTH); // Directly use Buffer returned by randomBytes
+  const cipher = crypto.createCipheriv(
+    "aes-256-gcm",
+    Buffer.from(keyHex, "hex"),
+    iv
+  );
+  let encrypted = cipher.update(plainText, "utf8", "hex");
+  encrypted += cipher.final("hex");
+  const authTag = cipher.getAuthTag();
+
+  // Return iv, encrypted data, and authTag as hex, combined in one line
+  return iv.toString("hex") + ":" + encrypted + ":" + authTag.toString("hex");
+}
+
+export function decrypt(text: string, keyHex: string = ENCRYPTION_KEY): string {
+  const [ivHex, encryptedHex, authTagHex] = text.split(":");
+  if (!ivHex || !encryptedHex || !authTagHex) {
+    throw new Error("Invalid or corrupted cipher format");
+  }
+
+  const iv = Buffer.from(ivHex, "hex");
+  const encryptedText = Buffer.from(encryptedHex, "hex");
+  const authTag = Buffer.from(authTagHex, "hex");
+
+  const decipher = crypto.createDecipheriv(
+    "aes-256-gcm",
+    Buffer.from(keyHex, "hex"),
+    iv
+  );
+  decipher.setAuthTag(authTag);
+
+  let decrypted = decipher.update(encryptedText, undefined, "utf8");
+  decrypted += decipher.final("utf8");
+
+  return decrypted.toString();
+}

packages/shared/src/env.ts

@@ -0,0 +1,13 @@
+import { z } from "zod";
+
+const EnvSchema = z.object({
+  ENCRYPTION_KEY: z
+    .string()
+    .length(
+      64,
+      "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32"
+    )
+    .optional(),
+});
+
+export const env = EnvSchema.parse(process.env);

packages/shared/src/features/evals/types.ts

@@ -0,0 +1,113 @@
+import z from "zod";
+import { ModelProvider, observationsTableCols } from "../..";
+
+export const langfuseObjects = [
+  "trace",
+  "span",
+  "generation",
+  "event",
+] as const;
+
+// variable mapping stored in the db for eval templates
+export const variableMapping = z
+  .object({
+    templateVariable: z.string(), // variable name in the template
+    // name of the observation to extract the variable from
+    // not required for trace, as we only have one.
+    objectName: z.string().nullish(),
+    langfuseObject: z.enum(langfuseObjects),
+    selectedColumnId: z.string(),
+  })
+  .refine(
+    (value) => value.langfuseObject === "trace" || value.objectName !== null,
+    {
+      message: "objectName is required for langfuseObjects other than trace",
+    }
+  );
+
+export const variableMappingList = z.array(variableMapping);
+
+export const wipVariableMapping = z.object({
+  templateVariable: z.string(),
+  objectName: z.string().nullish(),
+  langfuseObject: z.enum(langfuseObjects),
+  selectedColumnId: z.string().nullish(),
+});
+
+export const availableEvalVariables = [
+  {
+    id: "trace",
+    display: "Trace",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 't."metadata"',
+      },
+      { name: "Input", id: "input", internal: 't."input"' },
+      { name: "Output", id: "output", internal: 't."output"' },
+    ],
+  },
+  {
+    id: "span",
+    display: "Span",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'o."metadata"',
+      },
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  {
+    id: "generation",
+    display: "Generation",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'o."metadata"',
+      },
+      { name: "Input", id: "input", internal: 'o."input"' },
+      { name: "Output", id: "output", internal: 'o."output"' },
+    ],
+  },
+  { id: "event", display: "Event", availableColumns: observationsTableCols },
+];
+
+export const evalLLMModels = [
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-3.5-turbo",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+  {
+    provider: ModelProvider.OpenAI,
+    model: "gpt-4-turbo-preview",
+    temperature: 1,
+    maxTemperature: 2,
+    max_tokens: 256,
+    top_p: 1,
+  },
+] as const;
+
+export const EvalModelNames = z.enum(["gpt-3.5-turbo", "gpt-4-turbo-preview"]);
+
+export const OutputSchema = z.object({
+  reasoning: z.string(),
+  score: z.string(),
+});
+
+export enum EvalTargetObject {
+  Trace = "trace",
+}
+
+export const DEFAULT_TRACE_JOB_DELAY = 10_000;

packages/shared/src/filterToPrisma.ts

@@ -1,10 +1,10 @@
-import { type FilterState } from "@/src/features/filters/types";
-import { filterOperators } from "@/src/server/api/interfaces/filters";
+import { Prisma } from "@prisma/client";
 import {
+  ColumnDefinition,
   type TableNames as TableName,
-  type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { Prisma } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";
+import { FilterState } from "./types";
+import { filterOperators } from "./interfaces/filters";
 
 const operatorReplacements = {
   "any of": "IN",
@@ -27,7 +27,7 @@ const arrayOperatorReplacements = {
 export function tableColumnsToSqlFilterAndPrefix(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
   const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
   if (sql === Prisma.empty) {
@@ -43,14 +43,14 @@ export function tableColumnsToSqlFilterAndPrefix(
 export function tableColumnsToSqlFilter(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
-  table: TableName,
+  table: TableName
 ): Prisma.Sql {
   const internalFilters = filters.map((filter) => {
     // Get column definition to map column to internal name, e.g. "t.id"
     const col = tableColumns.find(
       (c) =>
         // TODO: Only use id instead of name
-        c.name === filter.column || c.id === filter.column,
+        c.name === filter.column || c.id === filter.column
     );
     if (!col) {
       console.error("Invalid filter column", filter.column);
@@ -72,13 +72,13 @@ export function tableColumnsToSqlFilter(
         ? Prisma.raw(
             arrayOperatorReplacements[
               filter.operator as keyof typeof arrayOperatorReplacements
-            ],
+            ]
           )
         : filter.operator in operatorReplacements
           ? Prisma.raw(
               operatorReplacements[
                 filter.operator as keyof typeof operatorReplacements
-              ],
+              ]
             )
           : Prisma.raw(filter.operator); //checked by zod
 
@@ -98,13 +98,13 @@ export function tableColumnsToSqlFilter(
         break;
       case "stringOptions":
         valuePrisma = Prisma.sql`(${Prisma.join(
-          filter.value.map((v) => Prisma.sql`${v}`),
+          filter.value.map((v) => Prisma.sql`${v}`)
         )})`;
         break;
       case "arrayOptions":
         valuePrisma = Prisma.sql`ARRAY[${Prisma.join(
           filter.value.map((v) => Prisma.sql`${v}`),
-          ", ",
+          ", "
         )}] `;
         break;
 
@@ -124,12 +124,12 @@ export function tableColumnsToSqlFilter(
       filter.type === "string" || filter.type === "stringObject"
         ? [
             ["contains", "does not contain", "ends with"].includes(
-              filter.operator,
+              filter.operator
             )
               ? Prisma.raw("'%' || ")
               : Prisma.empty,
             ["contains", "does not contain", "starts with"].includes(
-              filter.operator,
+              filter.operator
             )
               ? Prisma.raw(" || '%'")
               : Prisma.empty,
@@ -153,7 +153,7 @@ export function tableColumnsToSqlFilter(
 
 const castValueToPostgresTypes = (
   column: ColumnDefinition,
-  table: TableName,
+  table: TableName
 ) => {
   return column.name === "type" &&
     (table === "observations" ||
@@ -169,7 +169,7 @@ const dateOperators = filterOperators["datetime"];
 export const datetimeFilterToPrismaSql = (
   safeColumn: string,
   operator: (typeof dateOperators)[number],
-  value: Date,
+  value: Date
 ) => {
   if (!dateOperators.includes(operator)) {
     throw new Error("Invalid operator: " + operator);
@@ -179,6 +179,6 @@ export const datetimeFilterToPrismaSql = (
   }
 
   return Prisma.sql`AND ${Prisma.raw(safeColumn)} ${Prisma.raw(
-    operator,
+    operator
   )} ${value}::timestamp with time zone at time zone 'UTC'`;
 };

packages/shared/src/index.ts

@@ -1,9 +1,23 @@
-import { ModelUsageUnit } from "./constants";
-export { type DB } from "../prisma/generated/types";
-
-export * from "./auth/auth";
+export * from "./constants";
+export * from "./queues";
+export * from "./interfaces/exportTypes";
+export * from "./interfaces/filters";
+export * from "./interfaces/orderBy";
+export * from "./interfaces/tableDefinition";
+export * from "./types";
+export * from "./filterToPrisma";
+export * from "./tracesTable";
+export * from "./server/auth";
+export * from "./observationsTable";
+export { env } from "./env";
 
+// llm api
 export * from "./server/llm/types";
 export * from "./server/llm/fetchLLMCompletion";
 
-export { ModelUsageUnit };
+// evals
+export * from "./features/evals/types";
+
+// export db types only
+export * from "@prisma/client";
+export { type DB } from "../prisma/generated/types";

packages/shared/src/interfaces/filters.ts

@@ -20,7 +20,7 @@ export const filterOperators = {
 export const timeFilter = z.object({
   column: z.string(),
   operator: z.enum(filterOperators.datetime),
-  value: z.date(),
+  value: z.coerce.date(), // coerce required to parse stringified dates from the db in evals
   type: z.literal("datetime"),
 });
 export const stringFilter = z.object({

packages/shared/src/interfaces/tableDefinition.ts

@@ -6,27 +6,27 @@ export type OptionsDefinition = {
 export type ColumnDefinition =
   | {
       name: string;
-      id?: string; // TODO: Adopt for all tables
+      id: string;
       type: "number" | "string" | "datetime" | "boolean";
       internal: string;
     }
   | {
       name: string;
-      id?: string; // TODO: Adopt for all tables
+      id: string;
       type: "stringOptions";
       options: Array<OptionsDefinition>;
       internal: string;
     }
   | {
       name: string;
-      id?: string; // TODO: Adopt for all tables
+      id: string;
       type: "arrayOptions";
       options: Array<OptionsDefinition>;
       internal: string;
     }
   | {
       name: string;
-      id?: string; // TODO: Adopt for all tables
+      id: string;
       type: "stringObject" | "numberObject";
       internal: string;
       keyOptions?: Array<string>;
@@ -41,6 +41,7 @@ export const tableNames = [
   "traces_metrics",
   "traces_parent_observation_scores",
   "sessions",
+  "prompts",
 ] as const;
 
 export type TableNames = (typeof tableNames)[number];

packages/shared/src/observationsTable.ts

@@ -1,13 +1,13 @@
+import { ObservationLevel } from "@prisma/client";
 import {
   type OptionsDefinition,
   type ColumnDefinition,
-} from "@/src/server/api/interfaces/tableDefinition";
-import { ObservationLevel } from "@langfuse/shared/src/db";
+} from "./interfaces/tableDefinition";
 
 // to be used server side
 export const observationsTableCols: ColumnDefinition[] = [
   {
-    name: "Id",
+    name: "ID",
     id: "id",
     type: "string",
     internal: 'o."id"',
@@ -19,7 +19,7 @@ export const observationsTableCols: ColumnDefinition[] = [
     internal: 'o."name"',
     options: [], // to be added at runtime
   },
-  { name: "traceId", id: "traceId", type: "string", internal: 't."id"' },
+  { name: "Trace ID", id: "traceId", type: "string", internal: 't."id"' },
   {
     name: "Trace Name",
     id: "traceName",
@@ -27,7 +27,7 @@ export const observationsTableCols: ColumnDefinition[] = [
     internal: 't."name"',
     options: [], // to be added at runtime
   },
-  { name: "userId", id: "userId", type: "string", internal: 't."user_id"' },
+  { name: "User ID", id: "userId", type: "string", internal: 't."user_id"' },
   {
     name: "Start Time",
     id: "startTime",
@@ -40,14 +40,39 @@ export const observationsTableCols: ColumnDefinition[] = [
     type: "datetime",
     internal: 'o."end_time"',
   },
+  {
+    name: "Time To First Token (s)",
+    id: "timeToFirstToken",
+    type: "number",
+    internal: 'o."completion_start_time" - o."start_time"',
+  },
   {
     name: "Latency (s)",
     id: "latency",
     type: "number",
     internal: '"latency"',
   },
+  {
+    name: "Time Per Output Token (s)",
+    id: "timePerOutputToken",
+    type: "number",
+    internal: '"latency" / o."completion_tokens"',
+  },
+  {
+    name: "Input Cost ($)",
+    id: "inputCost",
+    type: "number",
+    internal: 'o."calculated_input_cost"',
+  },
+  {
+    name: "Output Cost ($)",
+    id: "outputCost",
+    type: "number",
+    internal: 'o."calculated_output_cost"',
+  },
   {
     name: "Total Cost ($)",
+    id: "totalCost",
     type: "number",
     internal: 'o."calculated_total_cost"',
   },
@@ -71,6 +96,30 @@ export const observationsTableCols: ColumnDefinition[] = [
     internal: 'o."model"',
     options: [], // to be added at runtime
   },
+  {
+    name: "Input Tokens",
+    id: "inputTokens",
+    type: "number",
+    internal: 'o."prompt_tokens"',
+  },
+  {
+    name: "Output Tokens",
+    id: "outputTokens",
+    type: "number",
+    internal: 'o."completion_tokens"',
+  },
+  {
+    name: "Total Tokens",
+    id: "totalTokens",
+    type: "number",
+    internal: 'o."total_tokens"',
+  },
+  {
+    name: "Usage",
+    id: "usage",
+    type: "number",
+    internal: 'o."total_tokens"',
+  },
   {
     name: "Metadata",
     id: "metadata",
@@ -96,6 +145,12 @@ export const observationsTableCols: ColumnDefinition[] = [
     internal: "p.name",
     options: [], // to be added at runtime
   },
+  {
+    name: "Prompt Version",
+    id: "promptVersion",
+    type: "number",
+    internal: "p.version",
+  },
 ];
 
 // to be used client side, insert options for use in filter-builder
@@ -109,7 +164,7 @@ export type ObservationOptions = {
 };
 
 export function observationsTableColsWithOptions(
-  options?: ObservationOptions,
+  options?: ObservationOptions
 ): ColumnDefinition[] {
   return observationsTableCols.map((col) => {
     if (col.id === "model") {

packages/shared/src/queues.ts

@@ -0,0 +1,36 @@
+import { z } from "zod";
+
+export const TraceUpsertEvent = z.object({
+  projectId: z.string(),
+  traceId: z.string(),
+});
+
+export const EvalExecutionEvent = z.object({
+  projectId: z.string(),
+  jobExecutionId: z.string(),
+});
+
+export enum QueueName {
+  TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
+}
+
+export enum QueueJobs {
+  TraceUpsert = "trace-upsert",
+  EvaluationExecution = "evaluation-execution-job",
+}
+
+export type TQueueJobTypes = {
+  [QueueName.TraceUpsert]: {
+    timestamp: Date;
+    id: string;
+    payload: z.infer<typeof TraceUpsertEvent>;
+    name: QueueJobs.TraceUpsert;
+  };
+  [QueueName.EvaluationExecution]: {
+    timestamp: Date;
+    id: string;
+    payload: z.infer<typeof EvalExecutionEvent>;
+    name: QueueJobs.EvaluationExecution;
+  };
+};

packages/shared/src/server/auth.ts

@@ -15,6 +15,7 @@ export function getDisplaySecretKey(secretKey: string) {
 }
 
 export async function hashSecretKey(key: string) {
+  // legacy, uses bcrypt, transformed into hashed key upon first use
   const hashedKey = await hash(key, 11);
   return hashedKey;
 }

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,25 +1,32 @@
-import {
-  type ChatMessage,
-  ChatMessageRole,
-  type ModelParams,
-  ModelProvider,
-} from "./types";
 import { ChatAnthropic } from "@langchain/anthropic";
 import {
   AIMessage,
   HumanMessage,
   SystemMessage,
 } from "@langchain/core/messages";
+import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 import {
   BytesOutputParser,
   StringOutputParser,
 } from "@langchain/core/output_parsers";
+import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI } from "@langchain/openai";
 
-console.log("fetchLLMCompletion.ts");
+import {
+  ChatMessage,
+  ChatMessageRole,
+  LLMFunctionCall,
+  ModelParams,
+  ModelProvider,
+} from "./types";
+import zodToJsonSchema from "zod-to-json-schema";
+import { JsonOutputFunctionsParser } from "langchain/output_parsers";
+
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
+  functionCall?: LLMFunctionCall;
+  callbacks?: BaseCallbackHandler[];
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -30,7 +37,7 @@ export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: true;
   }
-): Promise<ReadableStream<Uint8Array>>;
+): Promise<IterableReadableStream<Uint8Array>>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
@@ -38,10 +45,17 @@ export async function fetchLLMCompletion(
   }
 ): Promise<string>;
 
+export async function fetchLLMCompletion(
+  params: LLMCompletionParams & {
+    streaming: false;
+    functionCall: LLMFunctionCall;
+  }
+): Promise<unknown>;
+
 export async function fetchLLMCompletion(
   params: FetchLLMCompletionParams
-): Promise<string | ReadableStream<Uint8Array>> {
-  const { messages, modelParams, streaming } = params;
+): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
+  const { messages, modelParams, streaming, callbacks } = params;
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
       return new HumanMessage(message.content);
@@ -59,6 +73,7 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
+      callbacks,
     });
   } else {
     chatModel = new ChatOpenAI({
@@ -67,9 +82,26 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
+      callbacks,
     });
   }
 
+  console.log("Making LLM call with params: ", modelParams);
+
+  if (params.functionCall) {
+    const functionCallingModel = chatModel.bind({
+      functions: [
+        {
+          ...params.functionCall,
+          parameters: zodToJsonSchema(params.functionCall.parameters),
+        },
+      ],
+      function_call: { name: params.functionCall.name },
+    });
+    const outputParser = new JsonOutputFunctionsParser();
+    return await functionCallingModel.pipe(outputParser).invoke(finalMessages);
+  }
+
   if (streaming) {
     return chatModel.pipe(new BytesOutputParser()).stream(finalMessages);
   }

packages/shared/src/server/llm/types.ts

@@ -1,4 +1,7 @@
+import z from "zod";
+
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
+
 export type ChatMessage = {
   role: ChatMessageRole;
   content: string;
@@ -23,11 +26,13 @@ export type UIModelParams = Required<
 };
 
 // Generic config
-export type ModelConfig = {
-  max_tokens?: number;
-  temperature?: number;
-  top_p?: number;
-};
+export type ModelConfig = z.infer<typeof ZodModelConfig>;
+
+export const ZodModelConfig = z.object({
+  max_tokens: z.number().optional(),
+  temperature: z.number().optional(),
+  top_p: z.number().optional(),
+});
 
 // OpenAI
 export type OpenAIModelParams = {
@@ -72,3 +77,9 @@ export const supportedModels = {
   [ModelProvider.Anthropic]: anthropicModels,
   [ModelProvider.OpenAI]: openAIModels,
 } as const;
+
+export type LLMFunctionCall = {
+  name: string;
+  description: string;
+  parameters: z.ZodTypeAny; // this has to be a json schema for OpenAI
+};

packages/shared/src/tracesTable.ts

@@ -0,0 +1,136 @@
+import { ObservationLevel } from "@prisma/client";
+import { ColumnDefinition, OptionsDefinition } from ".";
+
+export const tracesTableCols: ColumnDefinition[] = [
+  { name: "⭐️", id: "bookmarked", type: "boolean", internal: "t.bookmarked" },
+  { name: "ID", id: "id", type: "string", internal: "t.id" },
+  {
+    name: "Name",
+    id: "name",
+    type: "stringOptions",
+    internal: 't."name"',
+    options: [], // to be filled in at runtime
+  },
+  {
+    name: "Timestamp",
+    id: "timestamp",
+    type: "datetime",
+    internal: 't."timestamp"',
+  },
+  { name: "User ID", id: "userId", type: "string", internal: 't."user_id"' },
+  {
+    name: "Session ID",
+    id: "sessionId",
+    type: "string",
+    internal: 't."session_id"',
+  },
+  {
+    name: "Input Tokens",
+    id: "inputTokens",
+    type: "number",
+    internal: 'tm."promptTokens"',
+  },
+  {
+    name: "Output Tokens",
+    id: "outputTokens",
+    type: "number",
+    internal: 'tm."completionTokens"',
+  },
+  {
+    name: "Total Tokens",
+    id: "totalTokens",
+    type: "number",
+    internal: 'tm."totalTokens"',
+  },
+  {
+    name: "Usage",
+    id: "usage",
+    type: "number",
+    internal: 'tm."totalTokens"',
+  },
+  {
+    name: "Metadata",
+    id: "metadata",
+    type: "stringObject",
+    internal: 't."metadata"',
+  },
+  {
+    name: "Scores",
+    id: "scores_avg",
+    type: "numberObject",
+    internal: "scores_avg",
+  },
+  {
+    name: "Latency (s)",
+    id: "latency",
+    type: "number",
+    internal: "tl.latency",
+  },
+  {
+    name: "Input Cost ($)",
+    id: "inputCost",
+    type: "number",
+    internal: '"calculatedInputCost"',
+  },
+  {
+    name: "Output Cost ($)",
+    id: "outputCost",
+    type: "number",
+    internal: '"calculatedOutputCost"',
+  },
+  {
+    name: "Total Cost ($)",
+    id: "totalCost",
+    type: "number",
+    internal: '"calculatedTotalCost"',
+  },
+  {
+    name: "Version",
+    id: "version",
+    type: "string",
+    internal: 't."version"',
+  },
+  {
+    name: "Release",
+    id: "release",
+    type: "string",
+    internal: 't."release"',
+  },
+  {
+    name: "Level",
+    id: "level",
+    type: "stringOptions",
+    internal: '"level"',
+    options: Object.values(ObservationLevel).map((value) => ({ value })),
+  },
+  {
+    name: "Tags",
+    id: "tags",
+    type: "arrayOptions",
+    internal: 't."tags"',
+    options: [], // to be filled in at runtime
+  },
+];
+
+export type TraceOptions = {
+  scores_avg: Array<string>;
+  name: Array<OptionsDefinition>;
+  tags: Array<OptionsDefinition>;
+};
+
+export function tracesTableColsWithOptions(
+  options?: TraceOptions
+): ColumnDefinition[] {
+  return tracesTableCols.map((col) => {
+    if (col.id === "scores_avg") {
+      return { ...col, keyOptions: options?.scores_avg ?? [] };
+    }
+    if (col.id === "name") {
+      return { ...col, options: options?.name ?? [] };
+    }
+    if (col.id === "tags") {
+      return { ...col, options: options?.tags ?? [] };
+    }
+    return col;
+  });
+}

packages/shared/src/types.ts

@@ -1,5 +1,5 @@
-import { type singleFilter } from "@/src/server/api/interfaces/filters";
 import { type z } from "zod";
+import { singleFilter } from "./interfaces/filters";
 
 // to be sent to the server
 export type FilterCondition = z.infer<typeof singleFilter>;
@@ -23,3 +23,11 @@ export type FilterOption = {
   value: string;
   count?: number;
 };
+
+export type TableName =
+  | "traces"
+  | "generations"
+  | "sessions"
+  | "scores"
+  | "prompts"
+  | "dashboard";

packages/shared/tsconfig.json

@@ -6,7 +6,8 @@
     "lib": ["ES2020"],
     "outDir": "./dist",
     "types": ["node"],
-    "target": "ES2020"
+    "target": "ES2020",
+    "rootDir": ".",
   },
   "include": ["."],
   "exclude": ["node_modules", "dist"]

pnpm-workspace.yaml

@@ -3,3 +3,4 @@ packages:
   - "web"
   - "worker"
   - "packages/**"
+  - "ee"

turbo.json

@@ -14,6 +14,10 @@
     "db:reset": {},
     "db:push": {},
     "db:seed": {},
+    "models:migrate": {
+      "dependsOn": ["db:generate"]
+    },
+    "db:seed:examples": {},
     "dev": {
       "cache": false,
       "persistent": true,

web/Dockerfile

@@ -5,10 +5,11 @@ FROM node:20-alpine AS alpine
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
 
 FROM alpine AS base
-RUN npm install pnpm turbo --global
+RUN npm install turbo --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable
+RUN corepack prepare pnpm@8.15.5 --activate
 
 
 FROM base AS pruner

web/generated/openapi-server/openapi.yml

@@ -152,25 +152,32 @@ paths:
           application/json:
             schema:
               $ref: '#/components/schemas/CreateDatasetRunItemRequest'
-  /api/public/datasets/{datasetName}:
+  /api/public/datasets:
     get:
-      description: Get a dataset and its items
-      operationId: datasets_get
+      description: Get all datasets
+      operationId: datasets_list
       tags:
         - Datasets
       parameters:
-        - name: datasetName
-          in: path
-          required: true
+        - name: page
+          in: query
+          required: false
           schema:
-            type: string
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          required: false
+          schema:
+            type: integer
+            nullable: true
       responses:
         '200':
           description: ''
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Dataset'
+                $ref: '#/components/schemas/PaginatedDatasets'
         '400':
           description: ''
           content:
@@ -197,7 +204,6 @@ paths:
             application/json:
               schema: {}
       security: *ref_0
-  /api/public/datasets:
     post:
       description: Create a dataset
       operationId: datasets_create
@@ -243,6 +249,51 @@ paths:
           application/json:
             schema:
               $ref: '#/components/schemas/CreateDatasetRequest'
+  /api/public/datasets/{datasetName}:
+    get:
+      description: Get a dataset and its items
+      operationId: datasets_get
+      tags:
+        - Datasets
+      parameters:
+        - name: datasetName
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Dataset'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
   /api/public/datasets/{datasetName}/runs/{runName}:
     get:
       description: Get a dataset run and its items
@@ -396,12 +447,14 @@ paths:
       parameters:
         - name: page
           in: query
+          description: page number, starts at 1
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: limit of items per page
           required: false
           schema:
             type: integer
@@ -429,6 +482,24 @@ paths:
             items:
               type: string
               nullable: true
+        - name: fromTimestamp
+          in: query
+          description: >-
+            Optional filter to only include traces on or after a certain
+            timestamp
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
+        - name: toTimestamp
+          in: query
+          description: Optional filter to only include traces before a certain timestamp
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
       responses:
         '200':
           description: ''
@@ -519,12 +590,16 @@ paths:
       parameters:
         - name: page
           in: query
+          description: Page number, starts at 1.
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: >-
+            Limit of items per page. If you encounter api issues due to too
+            large page sizes, try to reduce the limit.
           required: false
           schema:
             type: integer
@@ -559,6 +634,16 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: fromStartTime
+          in: query
+          description: >-
+            Retrieve only observations with a start_time greater than this
+            timestamp.
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
       responses:
         '200':
           description: ''
@@ -782,12 +867,16 @@ paths:
       parameters:
         - name: page
           in: query
+          description: Page number, starts at 1.
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: >-
+            Limit of items per page. If you encounter api issues due to too
+            large page sizes, try to reduce the limit.
           required: false
           schema:
             type: integer
@@ -804,6 +893,35 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: fromTimestamp
+          in: query
+          description: Retrieve only scores newer than this timestamp.
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
+        - name: source
+          in: query
+          description: Retrieve only scores from a specific source.
+          required: true
+          schema:
+            $ref: '#/components/schemas/ScoreSource'
+        - name: operator
+          in: query
+          description: Retrieve only scores with <operator> value.
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: value
+          in: query
+          description: Retrieve only scores with <operator> value.
+          required: false
+          schema:
+            type: number
+            format: double
+            nullable: true
       responses:
         '200':
           description: ''
@@ -1018,19 +1136,23 @@ paths:
       security: *ref_0
   /api/public/traces:
     get:
-      description: Get list of traces
+      description: Get list of traces.
       operationId: trace_list
       tags:
         - Trace
       parameters:
         - name: page
           in: query
+          description: Page number, starts at 1
           required: false
           schema:
             type: integer
             nullable: true
         - name: limit
           in: query
+          description: >-
+            Limit of items per page. If you encounter api issues due to too
+            large page sizes, try to reduce the limit.
           required: false
           schema:
             type: integer
@@ -1047,6 +1169,14 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: fromTimestamp
+          in: query
+          description: Retrieve only traces newer than this timestamp.
+          required: false
+          schema:
+            type: string
+            format: date-time
+            nullable: true
         - name: orderBy
           in: query
           description: >-
@@ -1394,14 +1524,19 @@ components:
         - value
         - source
         - timestamp
-    Dataset:
-      title: Dataset
+    DatasetCore:
+      title: DatasetCore
       type: object
       properties:
         id:
           type: string
         name:
           type: string
+        description:
+          type: string
+          nullable: true
+        metadata:
+          nullable: true
         projectId:
           type: string
         createdAt:
@@ -1410,6 +1545,36 @@ components:
         updatedAt:
           type: string
           format: date-time
+      required:
+        - id
+        - name
+        - projectId
+        - createdAt
+        - updatedAt
+    DatasetWithReferences:
+      title: DatasetWithReferences
+      type: object
+      properties:
+        items:
+          type: array
+          items:
+            type: string
+          description: list of dataset item ids
+        runs:
+          type: array
+          items:
+            type: string
+          description: list of dataset run names
+      required:
+        - items
+        - runs
+      allOf:
+        - $ref: '#/components/schemas/DatasetCore'
+    Dataset:
+      title: Dataset
+      type: object
+      description: Dataset including all items
+      properties:
         items:
           type: array
           items:
@@ -1419,13 +1584,10 @@ components:
           items:
             type: string
       required:
-        - id
-        - name
-        - projectId
-        - createdAt
-        - updatedAt
         - items
         - runs
+      allOf:
+        - $ref: '#/components/schemas/DatasetCore'
     DatasetItem:
       title: DatasetItem
       type: object
@@ -1434,14 +1596,22 @@ components:
           type: string
         status:
           $ref: '#/components/schemas/DatasetStatus'
-        input: {}
+        input:
+          nullable: true
         expectedOutput:
           nullable: true
+        metadata:
+          nullable: true
+        sourceTraceId:
+          type: string
+          nullable: true
         sourceObservationId:
           type: string
           nullable: true
         datasetId:
           type: string
+        datasetName:
+          type: string
         createdAt:
           type: string
           format: date-time
@@ -1451,8 +1621,8 @@ components:
       required:
         - id
         - status
-        - input
         - datasetId
+        - datasetName
         - createdAt
         - updatedAt
     DatasetRunItem:
@@ -1463,10 +1633,15 @@ components:
           type: string
         datasetRunId:
           type: string
+        datasetRunName:
+          type: string
         datasetItemId:
           type: string
+        traceId:
+          type: string
         observationId:
           type: string
+          nullable: true
         createdAt:
           type: string
           format: date-time
@@ -1476,8 +1651,9 @@ components:
       required:
         - id
         - datasetRunId
+        - datasetRunName
         - datasetItemId
-        - observationId
+        - traceId
         - createdAt
         - updatedAt
     DatasetRun:
@@ -1488,8 +1664,15 @@ components:
           type: string
         name:
           type: string
+        description:
+          type: string
+          nullable: true
+        metadata:
+          nullable: true
         datasetId:
           type: string
+        datasetName:
+          type: string
         createdAt:
           type: string
           format: date-time
@@ -1504,6 +1687,7 @@ components:
         - id
         - name
         - datasetId
+        - datasetName
         - createdAt
         - updatedAt
         - datasetRunItems
@@ -1550,42 +1734,82 @@ components:
       enum:
         - API
         - REVIEW
+        - EVAL
     CreateDatasetItemRequest:
       title: CreateDatasetItemRequest
       type: object
       properties:
         datasetName:
           type: string
-        input: {}
+        input:
+          nullable: true
         expectedOutput:
           nullable: true
+        metadata:
+          nullable: true
+        sourceTraceId:
+          type: string
+          nullable: true
+        sourceObservationId:
+          type: string
+          nullable: true
         id:
           type: string
           nullable: true
           description: Dataset items are upserted on their id
       required:
         - datasetName
-        - input
     CreateDatasetRunItemRequest:
       title: CreateDatasetRunItemRequest
       type: object
       properties:
         runName:
           type: string
+        runDescription:
+          type: string
+          nullable: true
+          description: Description of the run. If run exists, description will be updated.
+        metadata:
+          nullable: true
+          description: Metadata of the dataset run, updates run if run already exists
         datasetItemId:
           type: string
         observationId:
           type: string
+          nullable: true
+        traceId:
+          type: string
+          nullable: true
+          description: >-
+            traceId should always be provided. For compatibility with older SDK
+            versions it can also be inferred from the provided observationId.
       required:
         - runName
         - datasetItemId
-        - observationId
+    PaginatedDatasets:
+      title: PaginatedDatasets
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/DatasetWithReferences'
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
     CreateDatasetRequest:
       title: CreateDatasetRequest
       type: object
       properties:
         name:
           type: string
+        description:
+          type: string
+          nullable: true
+        metadata:
+          nullable: true
       required:
         - name
     HealthResponse:
@@ -2173,6 +2397,8 @@ components:
           type: string
         countTraces:
           type: integer
+        countObservations:
+          type: integer
         totalCost:
           type: number
           format: double
@@ -2183,6 +2409,7 @@ components:
       required:
         - date
         - countTraces
+        - countObservations
         - totalCost
         - usage
     UsageByModel:
@@ -2194,6 +2421,7 @@ components:
       properties:
         model:
           type: string
+          nullable: true
         inputUsage:
           type: integer
         outputUsage:
@@ -2201,7 +2429,6 @@ components:
         totalUsage:
           type: integer
       required:
-        - model
         - inputUsage
         - outputUsage
         - totalUsage
@@ -2254,6 +2481,50 @@ components:
         - name
     CreatePromptRequest:
       title: CreatePromptRequest
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/CreateChatPromptRequest'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/CreateTextPromptRequest'
+          required:
+            - type
+    CreateChatPromptRequest:
+      title: CreateChatPromptRequest
+      type: object
+      properties:
+        name:
+          type: string
+        isActive:
+          type: boolean
+          description: Should the prompt be promoted to production immediately?
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+        config:
+          nullable: true
+      required:
+        - name
+        - isActive
+        - prompt
+    CreateTextPromptRequest:
+      title: CreateTextPromptRequest
       type: object
       properties:
         name:
@@ -2271,20 +2542,75 @@ components:
         - prompt
     Prompt:
       title: Prompt
+      oneOf:
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - chat
+            - $ref: '#/components/schemas/ChatPrompt'
+          required:
+            - type
+        - type: object
+          allOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  enum:
+                    - text
+            - $ref: '#/components/schemas/TextPrompt'
+          required:
+            - type
+    BasePrompt:
+      title: BasePrompt
       type: object
       properties:
         name:
           type: string
         version:
           type: integer
-        prompt:
-          type: string
         config: {}
       required:
         - name
         - version
-        - prompt
         - config
+    ChatMessage:
+      title: ChatMessage
+      type: object
+      properties:
+        role:
+          type: string
+        content:
+          type: string
+      required:
+        - role
+        - content
+    TextPrompt:
+      title: TextPrompt
+      type: object
+      properties:
+        prompt:
+          type: string
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
+    ChatPrompt:
+      title: ChatPrompt
+      type: object
+      properties:
+        prompt:
+          type: array
+          items:
+            $ref: '#/components/schemas/ChatMessage'
+      required:
+        - prompt
+      allOf:
+        - $ref: '#/components/schemas/BasePrompt'
     CreateScoreRequest:
       title: CreateScoreRequest
       type: object

web/next.config.mjs

@@ -6,6 +6,25 @@ await import("./src/env.mjs");
 import { withSentryConfig } from "@sentry/nextjs";
 import { env } from "./src/env.mjs";
 
+/**
+ * CSP headers
+ * img-src https to allow loading images from SSO providers
+ */
+const cspHeader = `
+  default-src 'self' https://ph.langfuse.com https://*.posthog.com wss://*.crisp.chat https://*.crisp.chat;
+  script-src 'self' 'unsafe-eval' https://*.crisp.chat https://challenges.cloudflare.com https://ph.langfuse.com https://static.cloudflareinsights.com https://*.stripe.com;
+  style-src 'self' 'unsafe-inline' https://*.crisp.chat;
+  img-src 'self' https: blob: data:;
+  font-src 'self' https://*.crisp.chat;
+  frame-src 'self' https://challenges.cloudflare.com https://*.stripe.com;
+  worker-src 'self' blob:;
+  object-src 'none';
+  base-uri 'self';
+  form-action 'self';
+  frame-ancestors 'none';
+  ${env.LANGFUSE_CSP_ENFORCE_HTTPS === "true" ? "upgrade-insecure-requests; block-all-mixed-content;" : ""}
+`;
+
 /** @type {import("next").NextConfig} */
 const nextConfig = {
   transpilePackages: ["@langfuse/shared"],
@@ -32,27 +51,48 @@ const nextConfig = {
             key: "x-frame-options",
             value: "SAMEORIGIN",
           },
+          {
+            key: "X-Content-Type-Options",
+            value: "nosniff",
+          },
+          {
+            key: "Referrer-Policy",
+            value: "strict-origin-when-cross-origin",
+          },
+          {
+            key: "Permissions-Policy",
+            value: "autoplay=*, fullscreen=*, microphone=*",
+          },
+        ],
+      },
+      {
+        source: "/:path((?!api).*)*",
+        headers: [
+          {
+            key: "Content-Security-Policy",
+            value: cspHeader.replace(/\n/g, ""),
+          },
         ],
       },
       // Required to check authentication status from langfuse.com
       ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
         ? [
-            {
-              source: "/api/auth/session",
-              headers: [
-                {
-                  key: "Access-Control-Allow-Origin",
-                  value: "https://langfuse.com",
-                },
-                { key: "Access-Control-Allow-Credentials", value: "true" },
-                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-                {
-                  key: "Access-Control-Allow-Headers",
-                  value: "Content-Type, Authorization",
-                },
-              ],
-            },
-          ]
+          {
+            source: "/api/auth/session",
+            headers: [
+              {
+                key: "Access-Control-Allow-Origin",
+                value: "https://langfuse.com",
+              },
+              { key: "Access-Control-Allow-Credentials", value: "true" },
+              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+              {
+                key: "Access-Control-Allow-Headers",
+                value: "Content-Type, Authorization",
+              },
+            ],
+          },
+        ]
         : []),
     ];
   },

web/package.json

@@ -1,11 +1,14 @@
 {
   "name": "web",
-  "version": "2.12.0",
+  "version": "2.30.2",
   "private": true,
   "license": "MIT",
+  "engines": {
+    "node": ">=20"
+  },
   "scripts": {
-    "prebuild": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
-    "build": "dotenv -e ../.env -- next build",
+    "copy-openapi-spec": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
+    "build": "pnpm run copy-openapi-spec && INLINE_RUNTIME_CHUNK=false dotenv -e ../.env -- next build",
     "dev": "dotenv -e ../.env -- next dev",
     "lint": "dotenv -e ../.env -- next lint",
     "lint:fix": "dotenv -e ../.env -- next lint --fix",
@@ -15,21 +18,23 @@
     "test": "dotenv -e ../.env -- jest --runInBand",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
-    "release": "dotenv -e ../.env -- release-it",
     "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
-    "@aws-sdk/client-s3": "^3.507.0",
-    "@aws-sdk/lib-storage": "^3.515.0",
-    "@aws-sdk/s3-request-presigner": "^3.507.0",
-    "@headlessui/react": "^1.7.18",
-    "@heroicons/react": "^2.1.1",
+    "@aws-sdk/client-s3": "^3.550.0",
+    "@aws-sdk/lib-storage": "^3.550.0",
+    "@aws-sdk/s3-request-presigner": "^3.554.0",
+    "@codemirror/lang-json": "^6.0.1",
+    "@headlessui/react": "^1.7.19",
+    "@heroicons/react": "^2.1.3",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.1.8",
-    "@langchain/core": "^0.1.49",
-    "@langchain/openai": "^0.0.23",
+    "@langchain/anthropic": "^0.1.14",
+    "@langchain/core": "^0.1.58",
+    "@langchain/openai": "^0.0.28",
     "@langfuse/shared": "workspace:*",
+    "@langfuse/ee": "workspace:*",
+    "@marsidev/react-turnstile": "^0.5.4",
     "@next-auth/prisma-adapter": "^1.0.7",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
@@ -51,14 +56,14 @@
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@react-email/components": "^0.0.14",
+    "@react-email/components": "^0.0.16",
     "@react-email/render": "^0.0.12",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^7.102.1",
-    "@sentry/node": "^7.107.0",
-    "@sentry/profiling-node": "^7.102.1",
-    "@sentry/types": "^7.88.0",
+    "@sentry/nextjs": "^7.110.1",
+    "@sentry/node": "^7.110.1",
+    "@sentry/profiling-node": "^7.110.1",
+    "@sentry/types": "^7.110.1",
     "@t3-oss/env-nextjs": "^0.8.0",
     "@tailwindcss/forms": "^0.5.7",
     "@tanstack/react-query": "^4.36.1",
@@ -68,36 +73,38 @@
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
-    "ai": "^3.0.11",
+    "@uiw/codemirror-theme-github": "^4.21.25",
+    "@uiw/react-codemirror": "^4.21.25",
+    "ai": "^3.0.23",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
-    "cmdk": "^0.2.1",
-    "core-js": "^3.36.0",
+    "cmdk": "^1.0.0",
+    "core-js": "^3.36.1",
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
-    "js-tiktoken": "^1.0.10",
+    "js-tiktoken": "^1.0.11",
     "kysely": "^0.27.3",
-    "langchain": "^0.1.28",
+    "langchain": "^0.1.33",
     "lodash": "^4.17.21",
-    "lucide-react": "^0.330.0",
-    "next": "^14.1.0",
+    "lucide-react": "^0.364.0",
+    "next": "^14.2.1",
     "next-auth": "^4.24.7",
     "next-query-params": "^5.0.0",
-    "nodemailer": "^6.9.9",
-    "posthog-js": "^1.105.9",
+    "nodemailer": "^6.9.13",
+    "posthog-js": "^1.122.0",
     "posthog-node": "^3.6.3",
-    "prisma": "^5.11.0",
+    "prisma": "^5.12.1",
     "react": "18.2.0",
-    "react-day-picker": "^8.10.0",
+    "react-day-picker": "^8.10.1",
     "react-dom": "18.2.0",
-    "react-hook-form": "^7.50.1",
+    "react-hook-form": "^7.51.3",
     "react-icons": "^5.0.1",
-    "react-responsive": "^9.0.2",
-    "react18-json-view": "^0.2.8-canary.1",
-    "sonner": "^1.4.0",
+    "react-responsive": "^10.0.0",
+    "react18-json-view": "^0.2.8-canary.6",
+    "sonner": "^1.4.41",
     "superjson": "2.2.1",
     "tailwind-merge": "^2.2.1",
     "tailwindcss-animate": "^1.0.7",
@@ -108,68 +115,41 @@
   "devDependencies": {
     "@jedmao/location": "^3.0.0",
     "@mermaid-js/mermaid-cli": "^10.7.0",
-    "@playwright/test": "^1.41.2",
-    "@release-it/bumper": "^6.0.1",
+    "@playwright/test": "^1.43.1",
     "@testing-library/jest-dom": "^6.4.2",
-    "@testing-library/react": "^14.2.1",
+    "@testing-library/react": "^15.0.2",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
-    "@types/eslint": "^8.56.2",
+    "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
     "@types/lodash": "^4.14.202",
     "@types/node": "20.10.5",
     "@types/nodemailer": "^6.4.14",
-    "@types/react": "^18.2.55",
-    "@types/react-dom": "^18.2.19",
+    "@types/react": "^18.2.79",
+    "@types/react-dom": "^18.2.25",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "@typescript-eslint/parser": "^6.21.0",
-    "autoprefixer": "^10.4.17",
+    "@typescript-eslint/parser": "^7.7.0",
+    "autoprefixer": "^10.4.19",
     "dotenv-cli": "^7.3.0",
     "eslint": "^8.56.0",
-    "eslint-config-next": "^14.1.0",
+    "eslint-config-next": "^14.2.1",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "node-mocks-http": "^1.14.1",
-    "postcss": "^8.4.35",
+    "postcss": "^8.4.38",
     "prettier": "^3.2.5",
-    "prettier-plugin-tailwindcss": "^0.5.11",
-    "release-it": "^17.1.1",
-    "tailwindcss": "^3.4.1",
+    "prettier-plugin-tailwindcss": "^0.5.14",
+    "tailwindcss": "^3.4.3",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "tsx": "^4.7.1",
-    "typescript": "^5.3.3"
+    "tsx": "^4.7.2",
+    "typescript": "^5.4.5"
   },
   "ct3aMetadata": {
     "initVersion": "7.13.0"
   },
   "optionalDependencies": {
     "crisp-sdk-web": "^1.0.21"
-  },
-  "release-it": {
-    "git": {
-      "commitMessage": "chore: release v${version}",
-      "tagName": "v${version}"
-    },
-    "github": {
-      "release": true,
-      "web": true,
-      "autoGenerate": true,
-      "releaseName": "v${version}",
-      "comments": {
-        "submit": true,
-        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
-        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
-      }
-    },
-    "plugins": {
-      "@release-it/bumper": {
-        "out": {
-          "file": "./src/constants/VERSION.ts",
-          "type": "application/typescript"
-        }
-      }
-    }
   }
 }

web/public/.gitignore

@@ -0,0 +1 @@
+openapi*.yml

web/public/images/posthog-logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 140"><g fill-rule="nonzero" fill="none"><path d="M55.383 75.225c-1.874 3.748-7.222 3.748-9.096 0l-4.482-8.963a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.963Zm0 50.836c-1.874 3.747-7.222 3.747-9.096 0l-4.482-8.964a5.085 5.085 0 0 1 0-4.548l4.482-8.964c1.874-3.748 7.222-3.748 9.096 0l4.482 8.964a5.084 5.084 0 0 1 0 4.548l-4.482 8.964Z" fill="#1D4AFF"/><path d="M0 106.765c0-4.53 5.477-6.8 8.68-3.596l23.307 23.307c3.204 3.204.935 8.68-3.595 8.68H5.085A5.085 5.085 0 0 1 0 130.073v-23.307Zm0-24.55c0 1.35.536 2.643 1.49 3.596l47.856 47.856a5.086 5.086 0 0 0 3.595 1.49h26.286c4.53 0 6.799-5.477 3.595-8.681L8.681 52.334C5.477 49.131 0 51.4 0 55.93v26.286Zm0-50.834c0 1.348.536 2.642 1.49 3.595l98.69 98.691a5.086 5.086 0 0 0 3.596 1.49h26.285c4.53 0 6.8-5.477 3.596-8.681L8.681 1.5C5.477-1.704 0 .565 0 5.095v26.286Zm50.835 0c0 1.348.536 2.642 1.49 3.595l91.5 91.5c3.203 3.204 8.68.935 8.68-3.596V96.595a5.086 5.086 0 0 0-1.49-3.596l-91.5-91.5c-3.203-3.203-8.68-.934-8.68 3.596v26.286ZM110.35 1.5c-3.203-3.204-8.68-.935-8.68 3.595v26.286c0 1.348.536 2.642 1.49 3.595l40.664 40.665c3.204 3.204 8.68.935 8.68-3.596V45.76a5.086 5.086 0 0 0-1.489-3.596L110.35 1.5Z" fill="#F9BD2B"/><path d="m216.24 107.388-47.864-47.863c-3.204-3.204-8.681-.935-8.681 3.595v66.952a5.085 5.085 0 0 0 5.085 5.085h74.142a5.085 5.085 0 0 0 5.085-5.085v-6.097c0-2.809-2.286-5.052-5.07-5.414a39.27 39.27 0 0 1-22.698-11.173Zm-32.145 11.502a8.137 8.137 0 0 1-8.133-8.134 8.137 8.137 0 0 1 8.133-8.134 8.137 8.137 0 0 1 8.134 8.134 8.137 8.137 0 0 1-8.134 8.134Z" fill="#000"/><path d="M0 130.072a5.085 5.085 0 0 0 5.085 5.085h23.307c4.53 0 6.799-5.477 3.595-8.681L8.681 103.169C5.477 99.966 0 102.235 0 106.765v23.307Zm50.835-86.418L8.68 1.5C5.477-1.704 0 .565 0 5.095v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V43.654ZM8.68 52.334C5.477 49.131 0 51.4 0 55.93v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V94.489L8.68 52.334Z" fill="#1D4AFF"/><path d="M101.67 45.76a5.083 5.083 0 0 0-1.49-3.596L59.516 1.5c-3.204-3.204-8.681-.935-8.681 3.595v26.286c0 1.348.536 2.642 1.49 3.595l49.345 49.346V45.76Zm-50.835 89.397h28.392c4.53 0 6.799-5.477 3.595-8.681L50.835 94.489v40.668Zm0-91.503v38.562c0 1.348.536 2.642 1.49 3.595l49.345 49.346V96.595a5.084 5.084 0 0 0-1.49-3.596L50.835 43.654Z" fill="#F54E00"/><path d="M303.32 114.86h20.888V80.22h17.452c19.17 0 31.466-11.37 31.466-28.954 0-17.584-12.295-28.954-31.466-28.954h-38.34v92.547Zm20.888-52.488V40.16h15.337c7.932 0 12.692 4.23 12.692 11.105 0 6.876-4.76 11.106-12.692 11.106h-15.337Zm86.71 53.545c20.36 0 35.167-14.543 35.167-34.375 0-19.831-14.807-34.374-35.167-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375Zm-15.866-34.375c0-10.577 6.346-17.848 15.866-17.848 9.386 0 15.733 7.271 15.733 17.848 0 10.577-6.347 17.849-15.733 17.849-9.52 0-15.866-7.272-15.866-17.849Zm84.462 34.375c15.601 0 26.178-9.784 26.178-21.286 0-26.97-35.829-18.245-35.829-28.822 0-2.908 3.04-4.759 7.404-4.759 4.495 0 9.916 2.776 11.634 8.858l15.601-6.479c-3.04-9.65-14.279-16.261-27.896-16.261-14.676 0-23.798 8.725-23.798 19.17 0 25.252 35.3 18.245 35.3 28.69 0 3.702-3.437 6.214-8.594 6.214-7.403 0-12.56-5.156-14.146-11.37l-15.601 6.081c3.438 10.048 13.486 19.964 29.747 19.964Zm76.43-1.718-1.321-16.791c-2.248 1.19-5.157 1.586-7.536 1.586-4.76 0-7.933-3.437-7.933-9.387V64.355h16.13v-16.13h-16.13V28.923h-19.435v19.302h-10.577v16.13h10.577v27.764c0 16.13 10.974 23.798 25.384 23.798 3.967 0 7.669-.66 10.842-1.718Zm67.764-91.887v35.961h-36.755v-35.96h-20.89v92.546h20.89V76.122h36.755v38.737h21.021V22.312h-21.021Zm67.386 93.605c20.36 0 35.168-14.543 35.168-34.375 0-19.831-14.807-34.374-35.168-34.374-20.625 0-35.168 14.543-35.168 34.374 0 19.832 14.543 34.375 35.168 34.375ZM675.23 81.542c0-10.577 6.346-17.848 15.865-17.848 9.387 0 15.733 7.271 15.733 17.848 0 10.577-6.346 17.849-15.733 17.849-9.519 0-15.865-7.272-15.865-17.849Zm88.545 31.202c7.272 0 13.75-2.512 17.188-6.875v6.346c0 7.404-5.95 12.56-15.072 12.56-6.479 0-12.164-3.173-13.09-8.594l-17.715 2.777c2.38 12.56 15.204 21.022 30.805 21.022 20.492 0 34.11-12.032 34.11-29.88V48.225h-19.17v5.685c-3.57-4.098-9.652-6.742-17.452-6.742-18.51 0-30.144 12.692-30.144 32.788 0 20.096 11.634 32.788 30.54 32.788ZM752.14 79.956c0-9.916 5.817-16.262 14.807-16.262 9.123 0 14.94 6.346 14.94 16.262s-5.817 16.262-14.94 16.262c-8.99 0-14.807-6.346-14.807-16.262Z" fill="#000"/></g></svg>

web/public/openapi-client.yml

@@ -1,104 +0,0 @@
-openapi: 3.0.1
-info:
-  title: langfuse
-  version: ''
-paths:
-  /api/public/scores:
-    post:
-      description: Add a score to the database, upserts on id
-      operationId: score_create
-      tags:
-        - Score
-      parameters: []
-      responses:
-        '200':
-          description: ''
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Score'
-        '400':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '401':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '403':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-        '405':
-          description: ''
-          content:
-            application/json:
-              schema:
-                type: string
-      security:
-        - BearerAuth: []
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/CreateScoreRequest'
-components:
-  schemas:
-    CreateScoreRequest:
-      title: CreateScoreRequest
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        comment:
-          type: string
-      required:
-        - traceId
-        - name
-        - value
-    Score:
-      title: Score
-      type: object
-      properties:
-        id:
-          type: string
-        traceId:
-          type: string
-        name:
-          type: string
-        value:
-          type: number
-          format: double
-        observationId:
-          type: string
-        timestamp:
-          type: string
-          format: date-time
-        comment:
-          type: string
-      required:
-        - id
-        - traceId
-        - name
-        - value
-        - timestamp
-  securitySchemes:
-    BearerAuth:
-      type: http
-      scheme: bearer

web/sentry.client.config.ts

@@ -9,7 +9,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     // Set tracesSampleRate to 1.0 to capture 100%
     // of transactions for performance monitoring.
     // We recommend adjusting this value in production
-    tracesSampleRate: 0.3,
+    tracesSampleRate: 0.2,
 
     // Capture Replay for 100% of all sessions,
     // plus for 100% of sessions with an error

web/sentry.edge.config.ts

@@ -7,7 +7,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     // Set tracesSampleRate to 1.0 to capture 100%
     // of transactions for performance monitoring.
     // We recommend adjusting this value in production
-    tracesSampleRate: 0.3,
+    tracesSampleRate: 0.2,
 
     // ...
 

web/sentry.server.config.ts

@@ -12,7 +12,6 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     // Set tracesSampleRate to 1.0 to capture 100%
     // of transactions for performance monitoring.
     // We recommend adjusting this value in production
-    tracesSampleRate: 1.0,
     tracesSampler: (samplingContext: SamplingContext) => {
       if (
         samplingContext.request &&
@@ -20,9 +19,9 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
         samplingContext.request.url &&
         samplingContext.request.url.includes("api/trpc")
       ) {
-        return 1.0;
+        return 0.9;
       }
-      return 0.2;
+      return 0.15;
     },
 
     profilesSampleRate: 0.2, // Profiling sample rate is relative to tracesSampleRate

web/src/__e2e__/auth.spec.ts

@@ -10,7 +10,7 @@ test("should redirect to home if signed in", async ({ page }) => {
   await page.goto("/auth/sign-in");
   await page.fill('input[name="email"]', "demo@langfuse.com");
   await page.fill('input[type="password"]', "password");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-in-form"]');
 
   // wait 2 seconds
   await page.waitForTimeout(2000);
@@ -32,7 +32,7 @@ test("Successfully sign up & able to go to homepage", async ({ page }) => {
   await page.fill('input[name="name"]', "demo lang");
   await page.fill('input[name="email"]', randomEmailAddress());
   await page.fill('input[type="password"]', "password2");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // see get started page
   await expect(page).toHaveURL("/?getStarted=1");
@@ -45,7 +45,7 @@ test("Successfully sign up & able to go to homepage with uppercase email", async
   await page.fill('input[name="name"]', "demo lang");
   await page.fill('input[name="email"]', "A" + randomEmailAddress());
   await page.fill('input[type="password"]', "password3");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // see get started page
   await expect(page).toHaveURL("/?getStarted=1");
@@ -55,13 +55,13 @@ test("Signup input validation", async ({ page }) => {
   await page.goto("auth/sign-up");
   await page.fill('input[name="email"]', "notanemail");
   await page.fill('input[type="password"]', "shortPw");
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   await expect(page.getByText("Invalid email")).toBeVisible();
   await expect(
     page.getByText("Password must be at least 8 characters long"),
   ).toBeVisible();
-  await page.click('button[type="submit"]');
+  await page.click('button[data-testid="submit-email-password-sign-up-form"]');
   await page.waitForTimeout(2000);
   // don't see get started page
   await expect(page).not.toHaveURL("/?getStarted=1");