chore: release v2.2.0

Bumps the patches group with 3 updates: [posthog-node](https://github.com/PostHog/posthog-js-lite/tree/HEAD/posthog-node), [react-hook-form](https://github.com/react-hook-form/react-hook-form) and [@types/react](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/react).


Updates `posthog-node` from 3.6.1 to 3.6.2
- [Release notes](https://github.com/PostHog/posthog-js-lite/releases)
- [Changelog](https://github.com/PostHog/posthog-js-lite/blob/main/posthog-node/CHANGELOG.md)
- [Commits](https://github.com/PostHog/posthog-js-lite/commits/posthog-node-v3.6.2/posthog-node)

Updates `react-hook-form` from 7.50.0 to 7.50.1
- [Release notes](https://github.com/react-hook-form/react-hook-form/releases)
- [Changelog](https://github.com/react-hook-form/react-hook-form/blob/master/CHANGELOG.md)
- [Commits](https://github.com/react-hook-form/react-hook-form/compare/v7.50.0...v7.50.1)

Updates `@types/react` from 18.2.54 to 18.2.55
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/react)

---
updated-dependencies:
- dependency-name: posthog-node
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patches
- dependency-name: react-hook-form
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patches
- dependency-name: "@types/react"
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: patches
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.codespellrc

@@ -1,3 +1,4 @@
 [codespell]
 skip = .git,*.pdf,*.svg,package-lock.json,*.prisma
-ignore-words-list = afterall
+ignore-words-list = afterall,vertx
+

.env.dev.example

@@ -16,4 +16,17 @@ NEXTAUTH_SECRET="secret"
 
 # Langfuse experimental features
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
-SALT="salt"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# S3 storage
+S3_ENDPOINT=
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
+S3_BUCKET_NAME=
+S3_REGION=

.env.local.example

@@ -1,16 +1,14 @@
 # When adding additional environment variables, the schema in "/src/env.mjs"
 # should be updated accordingly.
+
 # Prisma
 # https://www.prisma.io/docs/reference/database-reference/connection-urls#env
 DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
 DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
 # Next Auth
-# You can generate a new secret on the command line with:
-# openssl rand -base64 32
-# https://next-auth.js.org/configuration/options#secret
 NEXTAUTH_SECRET="secret"
 NEXTAUTH_URL="http://localhost:3000"
 
 # feature flag to enable experimental features locally
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
-SALT="salt"
+SALT="salt"

.env.prod.example

@@ -1,31 +1,72 @@
-# Assuming deployment on Vercel with Postgres database on Supabase
-NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
-NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"
+# More information: https://langfuse.com/docs/deployment/self-host
+
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
 
 # Prisma
-
 # https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:[pw]@db.[db_id].supabase.co:5432/postgres"
-DATABASE_URL="postgres://postgres:[pw]@db.[db_id].supabase.co:6543/postgres?pgbouncer=true&connection_limit=1"
+# DATABASE_URL supports pooled connections, but then you need to set DIRECT_URL
+DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
+# DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
+# SHADOW_DATABASE_URL=
 
 # Next Auth
-
 # NEXTAUTH_URL does not need to be set when deploying on Vercel
-# NEXTAUTH_URL="http://localhost:3000"
-
-# AUTH_REDIRECT_PROXY_URL used to proxy oauth callbacks on e.g. preview deployments. optional.
-# AUTH_REDIRECT_PROXY_URL="https://example.com/api/auth"
+NEXTAUTH_URL="http://localhost:3000"
 
 # You can generate a new secret on the command line with:
 # openssl rand -base64 32
 # https://next-auth.js.org/configuration/options#secret
 NEXTAUTH_SECRET="secret"
 SALT="salt"
-# Sentry; set via Vercel integration
-# NEXT_PUBLIC_SENTRY_DSN=
-# NEXT_SENTRY_ORG=
-# NEXT_SENTRY_PROJECT=
-# SENTRY_AUTH_TOKEN=
+
+# Docker only, optional
+# PORT=3000
+# HOSTNAME=localhost
+
+# Default project, optional
+# LANGFUSE_DEFAULT_PROJECT_ID=
+# LANGFUSE_DEFAULT_PROJECT_ROLE=
+
+# Enable experimental features, optional
+# NEXT_PUBLIC_ENABLE_EXPERIMENTAL_FEATURES
+
+# Auth, optional configuration
+# AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
+# AUTH_DISABLE_USERNAME_PASSWORD=true
+
+# SSO, each group is optional
+# AUTH_GOOGLE_CLIENT_ID=
+# AUTH_GOOGLE_CLIENT_SECRET=
+# AUTH_GITHUB_CLIENT_ID=
+# AUTH_GITHUB_CLIENT_SECRET=
+# AUTH_AZURE_AD_CLIENT_ID=
+# AUTH_AZURE_AD_CLIENT_SECRET=
+# AUTH_AZURE_AD_TENANT_ID=
+
+# Transactional email, optional
+# Defines the email address to use as the from address.
+# EMAIL_FROM_ADDRESS=
+# Defines the connection url for smtp server. 
+# SMTP_CONNECTION_URL=
+
+# S3 storage, optional, used for exports from the UI
+# S3_ENDPOINT=
+# S3_ACCESS_KEY_ID=
+# S3_SECRET_ACCESS_KEY=
+# S3_BUCKET_NAME=
+# S3_REGION=
+
+
+
+
+
+### START Langfuse Cloud Config
+# Used for Langfuse Cloud deployments
+# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
+
+# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
+# NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"
 
 # LANGFUSE_TEAM_SLACK_WEBHOOK=
 # LANGFUSE_NEW_USER_SIGNUP_WEBHOOK=
@@ -34,13 +75,16 @@ SALT="salt"
 # NEXT_PUBLIC_POSTHOG_HOST=
 # NEXT_PUBLIC_POSTHOG_KEY=
 
-# Id of demo project to automatically assign new users to
+# Sentry
+# NEXT_PUBLIC_SENTRY_DSN=
+# NEXT_SENTRY_ORG=
+# NEXT_SENTRY_PROJECT=
+# SENTRY_AUTH_TOKEN=
+
+# Demo project that users can use to try the platform
 # NEXT_PUBLIC_DEMO_PROJECT_ID=
 
-# Auth, each group is optional
-AUTH_GOOGLE_CLIENT_ID=
-AUTH_GOOGLE_CLIENT_SECRET=
-AUTH_GITHUB_CLIENT_ID=
-AUTH_GITHUB_CLIENT_SECRET=
-# AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
-# AUTH_DISABLE_USERNAME_PASSWORD=true
+# Crisp chat
+# NEXT_PUBLIC_CRISP_WEBSITE_ID=
+
+### END Langfuse Cloud Config

.eslintrc.cjs

@@ -6,12 +6,17 @@ const config = {
   overrides: [
     {
       extends: [
-        "plugin:@typescript-eslint/recommended-requiring-type-checking",
+        "plugin:@typescript-eslint/recommended",
+        "plugin:@typescript-eslint/strict-type-checked",
       ],
       files: ["*.ts", "*.tsx"],
       parserOptions: {
         project: path.join(__dirname, "tsconfig.json"),
       },
+      rules: {
+        "@typescript-eslint/no-non-null-assertion": "off",
+        "@typescript-eslint/no-confusing-void-expression": "off",
+      },
     },
   ],
   parser: "@typescript-eslint/parser",
@@ -19,7 +24,8 @@ const config = {
     project: path.join(__dirname, "tsconfig.json"),
   },
   plugins: ["@typescript-eslint"],
-  extends: ["next/core-web-vitals", "plugin:@typescript-eslint/recommended"],
+  extends: ["next/core-web-vitals"],
+  ignorePatterns: ["generated/**/*"],
   rules: {
     "@typescript-eslint/consistent-type-imports": [
       "warn",
@@ -29,6 +35,12 @@ const config = {
       },
     ],
     "@typescript-eslint/no-unused-vars": ["warn", { argsIgnorePattern: "^_" }],
+    "react/jsx-key": [
+      "error",
+      {
+        warnOnDuplicates: true,
+      },
+    ],
   },
 };
 

.github/DISCUSSION_TEMPLATE/ideas.yml

@@ -0,0 +1,18 @@
+body:
+  - type: textarea
+    attributes:
+      label: Describe the feature or potential improvement
+      description: Please describe the change as clear and concise as possible. Remember to add context as to why you believe this is needed.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the change here. If your idea is related to any issues or discussions, link them here.
+  - type: checkboxes
+    id: contribute
+    attributes:
+      label: Contribute
+      description: Are you willing to contribute to the implementation of this idea?
+      options:
+        - label: Yes, I can implement this and raise a PR

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,7 @@
+contact_links:
+  - name: 💡 Feature Request
+    url: https://github.com/orgs/langfuse/discussions/new?category=ideas
+    about: Suggest any ideas you have using our discussion forums.
+  - name: 🤗 Get Help
+    url: https://github.com/orgs/langfuse/discussions/new?category=support
+    about: If you can’t get something to work the way you expect, open a question in our discussion forums.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,36 +0,0 @@
-# This template is heavily inspired by the Next.js's template:
-# See here: https://github.com/vercel/next.js/blob/canary/.github/ISSUE_TEMPLATE/3.feature_request.yml
-
-name: 🛠 Feature Request
-description: Create a feature request for the core packages
-title: "feat: "
-labels: ["✨ enhancement"]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thank you for taking the time to file a feature request. Please fill out this form as completely as possible.
-  - type: textarea
-    attributes:
-      label: Describe the feature you'd like to request
-      description: Please describe the feature as clear and concise as possible. Remember to add context as to why you believe this feature is needed.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Describe the solution you'd like to see
-      description: Please describe the solution you would like to see. Adding example usage is a good way to provide context.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Additional information
-      description: Add any other information related to the feature here. If your feature request is related to any issues or discussions, link them here.
-  - type: checkboxes
-    id: contribute
-    attributes:
-      label: Contribute
-      description: Are you willing to contribute to the implementation of this feature?
-      options:
-        - label: Yes, I can implement this and raise a PR
-    

.github/PULL_REQUEST_TEMPLATE.md

@@ -27,7 +27,7 @@ Fixes # (issue)
 
 <!-- Remove bullet points below that don't apply to you -->
 
-- I haven't read the [contributing guide](https://github.com/calcom/cal.com/blob/main/CONTRIBUTING.md)
+- I haven't read the [contributing guide](https://github.com/langfuse/langfuse/blob/main/CONTRIBUTING.md)
 - My code doesn't follow the style guidelines of this project (`npm run prettier`)
 - I haven't commented my code, particularly in hard-to-understand areas
 - I haven't checked if my PR needs changes to the documentation

.github/dependabot.yml

@@ -18,6 +18,9 @@ updates:
       - dependency-name: "@types/node"
       - dependency-name: "@trpc/*"
     groups:
+      sentry:
+        patterns:
+          - "@sentry/*"
       prisma:
         patterns:
           - "prisma"

.github/workflows/licencecheck.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - "main"
+  merge_group:
   pull_request:
     branches:
       - "main"

.github/workflows/pipeline.yml

@@ -7,6 +7,7 @@ on:
       - "main"
     tags:
       - "v*"
+  merge_group:
   pull_request:
     branches:
       - "main"
@@ -29,12 +30,47 @@ jobs:
       - name: lint
         run: npm run lint
 
+  test-docker-build:
+    runs-on: ubuntu-latest
+    env:
+      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+      NEXTAUTH_SECRET: "secret"
+      SALT: "salt"
+      NEXTAUTH_URL: "http://localhost:3030"
+      REGISTRY: ghcr.io
+      IMAGE_NAME: ${{ github.repository }}
+
+    steps:
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: false
+          build-args: |
+            DATABASE_URL=${{ env.DATABASE_URL }}
+            NEXTAUTH_SECRET=${{ env.NEXTAUTH_SECRET }}
+            NEXTAUTH_URL=${{ env.NEXTAUTH_URL }}
+            SALT=${{ env.SALT }}
+
   tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [18, 20]
     steps:
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
@@ -100,7 +136,7 @@ jobs:
   all-ci-passed:
     # This allows us to have a branch protection rule for tests and deploys with matrix
     runs-on: ubuntu-latest
-    needs: [lint, tests, e2e-tests]
+    needs: [lint, tests, e2e-tests, test-docker-build]
     if: always()
     steps:
       - name: Successful deploy

.vscode/settings.json

@@ -1,6 +1,6 @@
 {
   "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
+    "source.fixAll.eslint": "explicit"
   },
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "editor.formatOnSave": true,
@@ -28,5 +28,6 @@
   "docwriter.style": "JSDoc",
   "[prisma]": {
     "editor.defaultFormatter": "Prisma.prisma"
-  }
+  },
+  "eslint.lintTask.enable": true
 }

CONTRIBUTING.md

@@ -21,10 +21,87 @@ _Before making any significant changes, please [open an issue](https://github.co
 
 Once we've discussed your changes and you've got your code ready, make sure that tests are passing and open your pull request.
 
-## Getting started
-
 A good first step is to search for open [issues](https://github.com/langfuse/langfuse/issues). Issues are labeled, and some good issues to start with are labeled: [good first issue](https://github.com/langfuse/langfuse/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22).
 
+## Project Overview
+
+### Technologies we use
+
+- Application (this repository)
+  - NextJS 14, pages router
+  - NextAuth.js / Auth.js
+  - tRPC: Frontend APIs
+  - Prisma ORM
+  - Zod
+  - Tailwind CSS
+  - shadcn/ui tailwind components (using Radix and tanstack)
+  - Fern: generate OpenAPI spec and Pydantic models
+- JS SDK ([langfuse/langfuse-js](https://github.com/langfuse/langfuse-js))
+  - openapi-typescript to generated types based on OpenAPI spec
+- Python SDK ([langfuse/langfuse-python](https://github.com/langfuse/langfuse-python))
+  - Pydantic for input validation, models generated by fern
+
+### Architecture Overview
+
+```mermaid
+flowchart TB
+   subgraph s4["Clients"]
+      subgraph s2["langfuse/langfuse-python"]
+         Python["Python SDK"]
+         OAI["OpenAI drop-in replacement"] -->|extends| Python
+         LCPYTHON["Langchain Python Integration"] -->|extends| Python
+         Langflow -->|uses| LCPYTHON
+         LiteLLM -->|uses| Python
+      end
+      subgraph s3["langfuse/langfuse-js"]
+         JS["JS SDK"]
+         LCJS["Langchain JS Integration"]  -->|extends| JS
+         Flowise -->|uses| LCJS
+      end
+   end
+
+   DB[Postgres Database]
+	subgraph s1["Application (langfuse/langfuse)"]
+      API[Public HTTP API]
+      G[TRPC API]
+      I[NextAuth]
+      H[React Frontend]
+      Prisma[Prisma ORM]
+      H --> G
+      H --> I
+      G --> I
+      G --- Prisma
+      API --- Prisma
+      I --- Prisma
+	end
+   Prisma --- DB
+   JS --- API
+   Python --- API
+```
+
+### Database Overview
+
+The diagram below may not show all relationships if the foreign key is not defined in the database schema. For instance, `trace_id` in the `observation` table is not defined as a foreign key to the `trace` table to allow unordered ingestion of these objects, but it is still a foreign key in the application code.
+
+Full database schema: [prisma/schema.prisma](prisma/schema.prisma)
+
+<img src="./prisma/database.svg">
+
+### Infrastructure & Network Overview
+
+```mermaid
+flowchart LR
+   Browser ---|Web UI & TRPC API| App
+   Integrations/SDKs ---|Public HTTP API| App
+   subgraph i1["Application Network"]
+      App["Langfuse Application (Docker or Serverless)"]
+   end
+   subgraph i2["Database Network"]
+      DB["Postgres Database"]
+   end
+   App --- DB
+```
+
 ## Development Setup
 
 Requirements
@@ -69,10 +146,31 @@ Requirements
     npm run dev
    ```
 
+> [!NOTE]
+> If you frequently switch branches, use `npm run dx` instead of `npm run dev`. This command will install dependencies, reset the database (wipe and apply all migrations), and run the database seeder with example data before starting the development server.
+
 ## Commit messages
 
 On the main branch, we adhere to the best practices of [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/). All pull requests and branches are squash-merged to maintain a clean and readable history. This approach ensures the addition of a conventional commit message when merging contributions.
 
+## Test the public API
+
+The API is tested using Jest. With the development server running, you can run the tests with:
+
+Run all
+
+```bash
+npm run test
+```
+
+Run interactively in watch mode
+
+```bash
+npm run test:watch
+```
+
+These tests are also run in CI.
+
 ## CI/CD
 
 We use GitHub Actions for CI/CD, the configuration is in [`.github/workflows/pipeline.yml`](.github/workflows/pipeline.yml)
@@ -108,3 +206,5 @@ When a new release is tagged on the `main` branch (excluding prereleases), it tr
 ## License
 
 Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.
+
+When contributing to the Langfuse codebase, you need to agree to the [Contributor License Agreement](https://cla-assistant.io/langfuse/langfuse). You only need to do this once and the CLA bot will remind you if you haven't signed it yet.

README.md

@@ -13,6 +13,14 @@
    </div>
    </br>
    <div>
+       <a href="https://cloud.langfuse.com">
+         <strong>Langfuse Cloud »</strong>
+      </a>
+               </br>
+      <a href="https://langfuse.com/docs/deployment/self-host">
+         <strong>Self Host Langfuse »</strong>
+      </a>
+         </br>
       <a href="https://discord.gg/7NXusRtqYU">
          <strong>Join the Langfuse Discord »</strong>
       </a>
@@ -26,11 +34,12 @@
       <a href="https://github.com/langfuse/langfuse/issues/new?labels=%F0%9F%90%9E%E2%9D%94+unconfirmed+bug&projects=&template=bug_report.yml&title=bug%3A+">
          <strong>Report Bug</strong>
       </a> ·
-      <a href="https://github.com/langfuse/langfuse/issues/new?assignees=&labels=%E2%9C%A8+enhancement&projects=&template=feature_request.yml&title=feat%3A+">
+      <a href="https://github.com/orgs/langfuse/discussions/new?category=ideas">
          <strong>Feature Request</strong>
       </a>
    </div>
    </br>
+   </br>
    <div>
       <img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License">
       <a href="https://discord.gg/7NXusRtqYU"><img src="https://img.shields.io/discord/1111061815649124414?style=flat-square&logo=Discord&logoColor=white&label=Discord&color=%23434EE4" alt="Discord"></a>

SECURITY.md

@@ -0,0 +1,4 @@
+## Security Policy
+We strongly recommend using the latest version of Langfuse to receive all security updates.
+
+For more information, please refer to the [Data Security & Privacy](https://langfuse.com/docs/data-security-privacy) page in the documentation or contact security@langfuse.com.

docker-compose.build.yml

@@ -8,7 +8,7 @@ services:
         - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
         - NEXTAUTH_SECRET=mysecret
         - SALT=mysalt
-        - NEXTAUTH_URL=http:localhost:3000
+        - NEXTAUTH_URL=http://localhost:3000
     depends_on:
       - db
     ports:
@@ -18,7 +18,7 @@ services:
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt
-      - NEXTAUTH_URL=http:localhost:3000
+      - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}

docker-compose.yml

@@ -12,7 +12,7 @@ services:
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt
-      - NEXTAUTH_URL=http:localhost:3000
+      - NEXTAUTH_URL=http://localhost:3000
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}

entrypoint.sh

@@ -1,7 +1,22 @@
 #!/bin/sh
 
-# Set DIRECT_URL to the value of DATABASE_URL, required for migrations
-export DIRECT_URL=$DATABASE_URL
+# Check if DATABASE_URL is not set
+if [ -z "$DATABASE_URL" ]; then
+    # Check if all required variables are provided
+    if [ -n "$DATABASE_HOST" ] && [ -n "$DATABASE_USERNAME" ] && [ -n "$DATABASE_PASSWORD" ]  && [ -n "$DATABASE_NAME" ]; then
+        # Construct DATABASE_URL from the provided variables
+        DATABASE_URL="postgresql://${DATABASE_USERNAME}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}"
+        export DATABASE_URL
+    else
+        echo "Error: Required database environment variables are not set. Provide a postgres url for DATABASE_URL."
+        exit 1
+    fi
+fi
+
+# Set DIRECT_URL to the value of DATABASE_URL if it is not set, required for migrations
+if [ -z "$DIRECT_URL" ]; then
+    export DIRECT_URL=$DATABASE_URL
+fi
 
 # Apply migrations
 prisma migrate deploy

fern/apis/server/definition/api.yml

@@ -1,4 +1,11 @@
 name: langfuse
+docs: |
+  ## Authentication
+  Authenticate with the API using Basic Auth, get API keys in the project settings:
+
+  - username: Langfuse Public Key
+  - password: Langfuse Secret Key
+
 error-discrimination:
   strategy: status-code
 auth: basic

fern/apis/server/definition/commons.yml

@@ -1,32 +1,4 @@
 types:
-  # Create Bodies
-  CreateEventRequest:
-    properties:
-      id: optional<string>
-      traceId: optional<string>
-      name: optional<string>
-      startTime: optional<datetime>
-      metadata: optional<unknown>
-      input: optional<unknown>
-      output: optional<unknown>
-      level: optional<ObservationLevel>
-      statusMessage: optional<string>
-      parentObservationId: optional<string>
-      version: optional<string>
-  CreateSpanRequest:
-    extends: CreateEventRequest
-    properties:
-      endTime: optional<datetime>
-  CreateGenerationRequest:
-    extends: CreateSpanRequest
-    properties:
-      completionStartTime: optional<datetime>
-      model: optional<string>
-      modelParameters: optional<map<string, MapValue>>
-      prompt: optional<unknown>
-      completion: optional<unknown>
-      usage: optional<LLMUsage>
-
   # Objects
   Trace:
     properties:
@@ -35,10 +7,17 @@ types:
         docs: The unique identifier of a trace
       timestamp: datetime
       name: optional<string>
+      input: optional<unknown>
+      output: optional<unknown>
+      sessionId: optional<string>
       release: optional<string>
       version: optional<string>
       userId: optional<string>
       metadata: optional<unknown>
+      tags: optional<list<string>>
+      public:
+        type: optional<boolean>
+        docs: Public traces are accessible via url without login
   TraceWithDetails:
     extends: Trace
     properties:
@@ -51,8 +30,17 @@ types:
   TraceWithFullDetails:
     extends: Trace
     properties:
-      observations: list<Observation>
+      observations: list<ObservationsView>
       scores: list<Score>
+  Session:
+    properties:
+      id: string
+      createdAt: datetime
+      projectId: string
+  SessionWithTraces:
+    extends: Session
+    properties:
+      traces: list<Trace>
   Observation:
     properties:
       id: string
@@ -68,12 +56,32 @@ types:
       version: optional<string>
       metadata: optional<unknown>
       output: optional<unknown>
-      promptTokens: integer
-      completionTokens: integer
-      totalTokens: integer
+      usage: optional<Usage>
       level: ObservationLevel
       statusMessage: optional<string>
       parentObservationId: optional<string>
+      promptId: optional<string>
+
+  ObservationsView:
+    extends: Observation
+    properties:
+      modelId: optional<string>
+      inputPrice: optional<double>
+      outputPrice: optional<double>
+      totalPrice: optional<double>
+      calculatedInputCost: optional<double>
+      calculatedOutputCost: optional<double>
+      calculatedTotalCost: optional<double>
+
+  Usage:
+    properties:
+      input: optional<integer>
+      output: optional<integer>
+      total: optional<integer>
+      unit: optional<ModelUsageUnit>
+      inputCost: optional<double>
+      outputCost: optional<double>
+      totalCost: optional<double>
   Score:
     properties:
       id: string
@@ -87,7 +95,6 @@ types:
     properties:
       id: string
       name: string
-      status: DatasetStatus
       projectId: string
       createdAt: datetime
       updatedAt: datetime
@@ -121,6 +128,13 @@ types:
       datasetRunItems: list<DatasetRunItem>
 
   # Utilities
+  ModelUsageUnit:
+    enum:
+      - CHARACTERS
+      - TOKENS
+      - MILLISECONDS
+      - SECONDS
+      - IMAGES
   ObservationLevel:
     enum:
       - DEBUG
@@ -133,11 +147,7 @@ types:
       - optional<string>
       - optional<integer>
       - optional<boolean>
-  LLMUsage:
-    properties:
-      promptTokens: optional<integer>
-      completionTokens: optional<integer>
-      totalTokens: optional<integer>
+      - optional<list<string>>
   DatasetStatus:
     enum:
       - ACTIVE

fern/apis/server/definition/dataset-items.yml

@@ -7,13 +7,23 @@ service:
   endpoints:
     create:
       method: POST
-      docs: Create a dataset item
+      docs: Create a dataset item, upserts on id
       path: /dataset-items
       request: CreateDatasetItemRequest
       response: commons.DatasetItem
+    get:
+      docs: Get a specific dataset item
+      method: GET
+      path: /dataset-items/{id}
+      path-parameters:
+        id:
+          type: string
+      response: commons.DatasetItem
+
 types:
   CreateDatasetItemRequest:
     properties:
       datasetName: string
       input: unknown
       expectedOutput: optional<unknown>
+      id: optional<string>

fern/apis/server/definition/event.yml

@@ -1,13 +0,0 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
-imports:
-  commons: ./commons.yml
-service:
-  auth: true
-  base-path: /api/public
-  endpoints:
-    create:
-      docs: Add an event to the database
-      method: POST
-      path: /events
-      request: commons.CreateEventRequest
-      response: commons.Observation

fern/apis/server/definition/generations.yml

@@ -1,36 +0,0 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
-imports:
-  commons: ./commons.yml
-  pagination: ./utils/pagination.yml
-service:
-  auth: true
-  base-path: /api/public
-  endpoints:
-    log:
-      method: POST
-      path: /generations
-      request: commons.CreateGenerationRequest
-      response: commons.Observation
-    update:
-      method: PATCH
-      path: /generations
-      request: UpdateGenerationRequest
-      response: commons.Observation
-
-types:
-  UpdateGenerationRequest:
-    properties:
-      generationId: string
-      traceId: optional<string>
-      name: optional<string>
-      endTime: optional<datetime>
-      completionStartTime: optional<datetime>
-      model: optional<string>
-      modelParameters: optional<map<string, commons.MapValue>>
-      prompt: optional<unknown>
-      version: optional<string>
-      metadata: optional<unknown>
-      completion: optional<unknown>
-      usage: optional<commons.LLMUsage>
-      level: optional<commons.ObservationLevel>
-      statusMessage: optional<string>

fern/apis/server/definition/health.yml

@@ -7,9 +7,22 @@ service:
       docs: Check health of API and database
       method: GET
       path: /health
+      response: HealthResponse
       errors:
         - ServiceUnavailableError
 
+types:
+  HealthResponse:
+    properties:
+      version:
+        type: string
+        docs: Langfuse server version
+      status: string
+    examples:
+      - value:
+          version: 1.25.0
+          status: OK
+
 errors:
   ServiceUnavailableError:
     status-code: 503

fern/apis/server/definition/ingestion.yml

@@ -15,7 +15,7 @@ service:
         body:
           properties:
             batch: list<IngestionEvent>
-      response: commons.Score
+      response: IngestionResponse # will be reportet as 200 response, but endpoint returns 207
 
 types:
   IngestionEvent:
@@ -23,26 +23,206 @@ types:
     union:
       trace-create: TraceEvent
       score-create: ScoreEvent
-      observation-create: ObservationCreateEvent
-      observation-update: ObservationUpdateEvent
+      event-create: CreateEventEvent
+      generation-create: CreateGenerationEvent
+      generation-update: UpdateGenerationEvent
+      span-create: CreateSpanEvent
+      span-update: UpdateSpanEvent
+      sdk-log: SDKLogEvent
+
+      # both are legacy
+      observation-create: CreateObservationEvent
+      observation-update: UpdateObservationEvent
+
+  ObservationType:
+    enum:
+      - SPAN
+      - GENERATION
+      - EVENT
+
+  IngestionUsage:
+    discriminated: false
+    union:
+      - commons.Usage
+      - OpenAIUsage
+
+  OpenAIUsage:
+    properties:
+      promptTokens: optional<integer>
+      completionTokens: optional<integer>
+      totalTokens: optional<integer>
+
+  OptionalObservationBody:
+    properties:
+      traceId: optional<string>
+      name: optional<string>
+      startTime: optional<datetime>
+      metadata: optional<unknown>
+      input: optional<unknown>
+      output: optional<unknown>
+      level: optional<commons.ObservationLevel>
+      statusMessage: optional<string>
+      parentObservationId: optional<string>
+      version: optional<string>
+
+  CreateEventBody:
+    extends: OptionalObservationBody
+    properties:
+      id: optional<string>
+
+  UpdateEventBody:
+    extends: OptionalObservationBody
+    properties:
+      id: string
+
+  CreateSpanBody:
+    extends: CreateEventBody
+    properties:
+      endTime: optional<datetime>
+
+  UpdateSpanBody:
+    extends: UpdateEventBody
+    properties:
+      endTime: optional<datetime>
+
+  CreateGenerationBody:
+    extends: CreateSpanBody
+    properties:
+      completionStartTime: optional<datetime>
+      model: optional<string>
+      modelParameters: optional<map<string, commons.MapValue>>
+      usage: optional<IngestionUsage>
+      promptName: optional<string>
+      promptVersion: optional<integer>
+
+  UpdateGenerationBody:
+    extends: UpdateSpanBody
+    properties:
+      completionStartTime: optional<datetime>
+      model: optional<string>
+      modelParameters: optional<map<string, commons.MapValue>>
+      usage: optional<IngestionUsage>
+      promptName: optional<string>
+      promptVersion: optional<integer>
+
+  ObservationBody:
+    properties:
+      id: optional<string>
+      traceId: optional<string>
+      type: ObservationType
+      name: optional<string>
+      startTime: optional<datetime>
+      endTime: optional<datetime>
+      completionStartTime: optional<datetime>
+      model: optional<string>
+      modelParameters: optional<map<string, commons.MapValue>>
+      input: optional<unknown>
+      version: optional<string>
+      metadata: optional<unknown>
+      output: optional<unknown>
+      usage: optional<commons.Usage>
+      level: optional<commons.ObservationLevel>
+      statusMessage: optional<string>
+      parentObservationId: optional<string>
+
+  TraceBody:
+    properties:
+      id: optional<string>
+      name: optional<string>
+      userId: optional<string>
+      input: optional<unknown>
+      output: optional<unknown>
+      sessionId: optional<string>
+      release: optional<string>
+      version: optional<string>
+      metadata: optional<unknown>
+      tags: optional<list<string>>
+      public:
+        type: optional<boolean>
+        docs: Make trace publicly accessible via url
+
+  SDKLogBody:
+    properties:
+      log: unknown
+
+  ScoreBody:
+    properties:
+      id: optional<string>
+      traceId: string
+      name: string
+      value: double
+      observationId: optional<string>
+      comment: optional<string>
+
+  BaseEvent:
+    properties:
+      id: string
+      timestamp: string
+      metadata: unknown
 
   TraceEvent:
+    extends: BaseEvent
     properties:
-      id: string
-      timestamp: string
-      body: commons.Trace
-  ObservationCreateEvent:
+      body: TraceBody
+
+  CreateObservationEvent:
+    extends: BaseEvent
     properties:
-      id: string
-      timestamp: string
-      body: commons.Observation
-  ObservationUpdateEvent:
+      body: ObservationBody
+
+  UpdateObservationEvent:
+    extends: BaseEvent
     properties:
-      id: string
-      timestamp: string
-      body: commons.Observation
+      body: ObservationBody
+
   ScoreEvent:
+    extends: BaseEvent
+    properties:
+      body: ScoreBody
+
+  SDKLogEvent:
+    extends: BaseEvent
+    properties:
+      body: SDKLogBody
+
+  CreateGenerationEvent:
+    extends: BaseEvent
+    properties:
+      body: CreateGenerationBody
+
+  UpdateGenerationEvent:
+    extends: BaseEvent
+    properties:
+      body: UpdateGenerationBody
+
+  CreateSpanEvent:
+    extends: BaseEvent
+    properties:
+      body: CreateSpanBody
+
+  UpdateSpanEvent:
+    extends: BaseEvent
+    properties:
+      body: UpdateSpanBody
+
+  CreateEventEvent:
+    extends: BaseEvent
+    properties:
+      body: CreateEventBody
+
+  IngestionSuccess:
     properties:
       id: string
-      timestamp: string
-      body: commons.Score
+      status: integer
+
+  IngestionError:
+    properties:
+      id: string
+      status: integer
+      message: optional<string>
+      error: optional<unknown>
+
+  IngestionResponse:
+    properties:
+      successes: list<IngestionSuccess>
+      errors: list<IngestionError>

fern/apis/server/definition/observations.yml

@@ -14,7 +14,7 @@ service:
         observationId:
           type: string
           docs: The unique langfuse identifier of an observation, can be an event, span or generation
-      response: commons.Observation
+      response: commons.ObservationsView
     getMany:
       docs: Get a list of observations
       method: GET
@@ -29,10 +29,15 @@ service:
           type: optional<string>
           traceId: optional<string>
           parentObservationId: optional<string>
-      response: Observations
+      response: ObservationsViews
 
 types:
   Observations:
     properties:
       data: list<commons.Observation>
       meta: pagination.MetaResponse
+
+  ObservationsViews:
+    properties:
+      data: list<commons.ObservationsView>
+      meta: pagination.MetaResponse

fern/apis/server/definition/prompts.yml

@@ -0,0 +1,36 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    get:
+      docs: Get a specific prompt
+      method: GET
+      path: /prompts
+      request:
+        name: GetParameterRequest
+        query-parameters:
+          name: string
+          version: optional<integer>
+      response: Prompt
+    create:
+      docs: Create a specific prompt
+      method: POST
+      path: /prompts
+      request: CreatePromptRequest
+      response: Prompt
+
+types:
+  CreatePromptRequest:
+    properties:
+      name: string
+      isActive: boolean
+      prompt: string
+  Prompt:
+    properties:
+      name: string
+      version: integer
+      prompt: string

fern/apis/server/definition/sessions.yml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  pagination: ./utils/pagination.yml
+  commons: ./commons.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    get:
+      docs: Get a session
+      method: GET
+      path: /sessions/{sessionId}
+      path-parameters:
+        sessionId:
+          type: string
+          docs: The unique id of a session
+      response: commons.SessionWithTraces

fern/apis/server/definition/span.yml

@@ -1,33 +0,0 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
-imports:
-  commons: ./commons.yml
-service:
-  auth: true
-  base-path: /api/public
-  endpoints:
-    create:
-      docs: Add a span to the database
-      method: POST
-      path: /spans
-      request: commons.CreateSpanRequest
-      response: commons.Observation
-    update:
-      docs: Update a span to the database
-      method: PATCH
-      path: /spans
-      request: UpdateSpanRequest
-      response: commons.Observation
-
-types:
-  UpdateSpanRequest:
-    properties:
-      spanId: string
-      traceId: optional<string>
-      endTime: optional<datetime>
-      name: optional<string>
-      metadata: optional<unknown>
-      input: optional<unknown>
-      output: optional<unknown>
-      level: optional<commons.ObservationLevel>
-      version: optional<string>
-      statusMessage: optional<string>

fern/apis/server/definition/trace.yml

@@ -6,12 +6,6 @@ service:
   auth: true
   base-path: /api/public
   endpoints:
-    create:
-      docs: Add a trace to the database
-      method: POST
-      path: /traces
-      request: CreateTraceRequest
-      response: commons.Trace
     get:
       docs: Get a specific trace
       method: GET
@@ -32,19 +26,20 @@ service:
           limit: optional<integer>
           userId: optional<string>
           name: optional<string>
+          orderBy:
+            type: string
+            docs: Format of the string sort_by=timestamp.asc (id, timestamp, name, userId, release, version, public, bookmarked, sessionId)
+          tags:
+            type: optional<string>
+            allow-multiple: true
+            docs: Only traces that include all of these tags will be returned.
       response: Traces
 
 types:
-  CreateTraceRequest:
-    properties:
-      id: optional<string>
-      name: optional<string>
-      userId: optional<string>
-      externalId: optional<string>
-      release: optional<string>
-      version: optional<string>
-      metadata: optional<unknown>
   Traces:
     properties:
       data: list<commons.TraceWithDetails>
       meta: pagination.MetaResponse
+  Sort:
+    properties:
+      id: string

fern/apis/server/generators.yml

@@ -11,13 +11,14 @@ groups:
           namespaceExport: Langfuse
           allowCustomFetcher: true
       - name: fernapi/fern-python-sdk
-        version: 0.6.6
+        version: 0.7.3
         output:
           location: local-file-system
           path: ../../../generated/python
         config:
-          namespaceExport: Langfuse
-          allowCustomFetcher: true
+          client_class_name: FernLangfuse
+          pydantic_config:
+            require_optional_fields: false
       - name: fernapi/fern-typescript-node-sdk
         version: 0.7.1
         output:

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "finto",
-  "version": "0.15.3"
+  "version": "0.16.36"
 }

generated/postman/collection.json

@@ -2,7 +2,7 @@
   "info": {
     "name": "Langfuse",
     "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-    "description": null
+    "description": "## Authentication\n\nAuthenticate with the API using Basic Auth, get API keys in the project settings:\n\n- username: Langfuse Public Key\n- password: Langfuse Secret Key"
   },
   "variable": [
     {
@@ -46,7 +46,7 @@
           "_type": "endpoint",
           "name": "Create",
           "request": {
-            "description": "Create a dataset item",
+            "description": "Create a dataset item, upserts on id",
             "url": {
               "raw": "{{baseUrl}}/api/public/dataset-items",
               "host": [
@@ -65,7 +65,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\"\n}",
+              "raw": "{\n    \"datasetName\": \"example\",\n    \"input\": \"UNKNOWN\",\n    \"expectedOutput\": \"UNKNOWN\",\n    \"id\": \"example\"\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -74,6 +74,38 @@
             }
           },
           "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Get",
+          "request": {
+            "description": "Get a specific dataset item",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/dataset-items/:id",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "dataset-items",
+                ":id"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "id",
+                  "value": "",
+                  "description": null
+                }
+              ]
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
         }
       ]
     },
@@ -228,119 +260,6 @@
         }
       ]
     },
-    {
-      "_type": "container",
-      "description": null,
-      "name": "Event",
-      "item": [
-        {
-          "_type": "endpoint",
-          "name": "Create",
-          "request": {
-            "description": "Add an event to the database",
-            "url": {
-              "raw": "{{baseUrl}}/api/public/events",
-              "host": [
-                "{{baseUrl}}"
-              ],
-              "path": [
-                "api",
-                "public",
-                "events"
-              ],
-              "query": [],
-              "variable": []
-            },
-            "header": [],
-            "method": "POST",
-            "auth": null,
-            "body": {
-              "mode": "raw",
-              "raw": "{\n    \"id\": \"example\",\n    \"traceId\": \"example\",\n    \"name\": \"example\",\n    \"startTime\": \"1994-11-05T13:15:30Z\",\n    \"metadata\": \"UNKNOWN\",\n    \"input\": \"UNKNOWN\",\n    \"output\": \"UNKNOWN\",\n    \"level\": \"DEBUG\",\n    \"statusMessage\": \"example\",\n    \"parentObservationId\": \"example\",\n    \"version\": \"example\"\n}",
-              "options": {
-                "raw": {
-                  "language": "json"
-                }
-              }
-            }
-          },
-          "response": []
-        }
-      ]
-    },
-    {
-      "_type": "container",
-      "description": null,
-      "name": "Generations",
-      "item": [
-        {
-          "_type": "endpoint",
-          "name": "Log",
-          "request": {
-            "description": null,
-            "url": {
-              "raw": "{{baseUrl}}/api/public/generations",
-              "host": [
-                "{{baseUrl}}"
-              ],
-              "path": [
-                "api",
-                "public",
-                "generations"
-              ],
-              "query": [],
-              "variable": []
-            },
-            "header": [],
-            "method": "POST",
-            "auth": null,
-            "body": {
-              "mode": "raw",
-              "raw": "{\n    \"completionStartTime\": \"1994-11-05T13:15:30Z\",\n    \"model\": \"example\",\n    \"modelParameters\": {\n        \"example\": \"example\"\n    },\n    \"prompt\": \"UNKNOWN\",\n    \"completion\": \"UNKNOWN\",\n    \"usage\": {\n        \"promptTokens\": 0,\n        \"completionTokens\": 0,\n        \"totalTokens\": 0\n    },\n    \"endTime\": \"1994-11-05T13:15:30Z\",\n    \"id\": \"example\",\n    \"traceId\": \"example\",\n    \"name\": \"example\",\n    \"startTime\": \"1994-11-05T13:15:30Z\",\n    \"metadata\": \"UNKNOWN\",\n    \"input\": \"UNKNOWN\",\n    \"output\": \"UNKNOWN\",\n    \"level\": \"DEBUG\",\n    \"statusMessage\": \"example\",\n    \"parentObservationId\": \"example\",\n    \"version\": \"example\"\n}",
-              "options": {
-                "raw": {
-                  "language": "json"
-                }
-              }
-            }
-          },
-          "response": []
-        },
-        {
-          "_type": "endpoint",
-          "name": "Update",
-          "request": {
-            "description": null,
-            "url": {
-              "raw": "{{baseUrl}}/api/public/generations",
-              "host": [
-                "{{baseUrl}}"
-              ],
-              "path": [
-                "api",
-                "public",
-                "generations"
-              ],
-              "query": [],
-              "variable": []
-            },
-            "header": [],
-            "method": "PATCH",
-            "auth": null,
-            "body": {
-              "mode": "raw",
-              "raw": "{\n    \"generationId\": \"example\",\n    \"traceId\": \"example\",\n    \"name\": \"example\",\n    \"endTime\": \"1994-11-05T13:15:30Z\",\n    \"completionStartTime\": \"1994-11-05T13:15:30Z\",\n    \"model\": \"example\",\n    \"modelParameters\": {\n        \"example\": \"example\"\n    },\n    \"prompt\": \"UNKNOWN\",\n    \"version\": \"example\",\n    \"metadata\": \"UNKNOWN\",\n    \"completion\": \"UNKNOWN\",\n    \"usage\": {\n        \"promptTokens\": 0,\n        \"completionTokens\": 0,\n        \"totalTokens\": 0\n    },\n    \"level\": \"DEBUG\",\n    \"statusMessage\": \"example\"\n}",
-              "options": {
-                "raw": {
-                  "language": "json"
-                }
-              }
-            }
-          },
-          "response": []
-        }
-      ]
-    },
     {
       "_type": "container",
       "description": null,
@@ -401,7 +320,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"body\": {\n                \"id\": \"example\",\n                \"timestamp\": \"1994-11-05T13:15:30Z\",\n                \"name\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"userId\": \"example\",\n                \"metadata\": \"UNKNOWN\"\n            }\n        }\n    ]\n}",
+              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"body\": {\n                \"id\": \"example\",\n                \"name\": \"example\",\n                \"userId\": \"example\",\n                \"input\": \"UNKNOWN\",\n                \"output\": \"UNKNOWN\",\n                \"sessionId\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"metadata\": \"UNKNOWN\",\n                \"tags\": [\n                    \"example\"\n                ],\n                \"public\": true\n            },\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"metadata\": \"UNKNOWN\"\n        }\n    ]\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -545,6 +464,82 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Prompts",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Get",
+          "request": {
+            "description": "Get a specific prompt",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/prompts?name=&version=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "prompts"
+              ],
+              "query": [
+                {
+                  "key": "name",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "version",
+                  "value": "",
+                  "description": null
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Create",
+          "request": {
+            "description": "Create a specific prompt",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/prompts",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "prompts"
+              ],
+              "query": [],
+              "variable": []
+            },
+            "header": [],
+            "method": "POST",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"name\": \"example\",\n    \"isActive\": true,\n    \"prompt\": \"example\"\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,
@@ -634,71 +629,37 @@
     {
       "_type": "container",
       "description": null,
-      "name": "Span",
+      "name": "Sessions",
       "item": [
         {
           "_type": "endpoint",
-          "name": "Create",
+          "name": "Get",
           "request": {
-            "description": "Add a span to the database",
+            "description": "Get a session",
             "url": {
-              "raw": "{{baseUrl}}/api/public/spans",
+              "raw": "{{baseUrl}}/api/public/sessions/:sessionId",
               "host": [
                 "{{baseUrl}}"
               ],
               "path": [
                 "api",
                 "public",
-                "spans"
+                "sessions",
+                ":sessionId"
               ],
               "query": [],
-              "variable": []
+              "variable": [
+                {
+                  "key": "sessionId",
+                  "value": "",
+                  "description": "The unique id of a session"
+                }
+              ]
             },
             "header": [],
-            "method": "POST",
+            "method": "GET",
             "auth": null,
-            "body": {
-              "mode": "raw",
-              "raw": "{\n    \"endTime\": \"1994-11-05T13:15:30Z\",\n    \"id\": \"example\",\n    \"traceId\": \"example\",\n    \"name\": \"example\",\n    \"startTime\": \"1994-11-05T13:15:30Z\",\n    \"metadata\": \"UNKNOWN\",\n    \"input\": \"UNKNOWN\",\n    \"output\": \"UNKNOWN\",\n    \"level\": \"DEBUG\",\n    \"statusMessage\": \"example\",\n    \"parentObservationId\": \"example\",\n    \"version\": \"example\"\n}",
-              "options": {
-                "raw": {
-                  "language": "json"
-                }
-              }
-            }
-          },
-          "response": []
-        },
-        {
-          "_type": "endpoint",
-          "name": "Update",
-          "request": {
-            "description": "Update a span to the database",
-            "url": {
-              "raw": "{{baseUrl}}/api/public/spans",
-              "host": [
-                "{{baseUrl}}"
-              ],
-              "path": [
-                "api",
-                "public",
-                "spans"
-              ],
-              "query": [],
-              "variable": []
-            },
-            "header": [],
-            "method": "PATCH",
-            "auth": null,
-            "body": {
-              "mode": "raw",
-              "raw": "{\n    \"spanId\": \"example\",\n    \"traceId\": \"example\",\n    \"endTime\": \"1994-11-05T13:15:30Z\",\n    \"name\": \"example\",\n    \"metadata\": \"UNKNOWN\",\n    \"input\": \"UNKNOWN\",\n    \"output\": \"UNKNOWN\",\n    \"level\": \"DEBUG\",\n    \"version\": \"example\",\n    \"statusMessage\": \"example\"\n}",
-              "options": {
-                "raw": {
-                  "language": "json"
-                }
-              }
-            }
+            "body": null
           },
           "response": []
         }
@@ -709,39 +670,6 @@
       "description": null,
       "name": "Trace",
       "item": [
-        {
-          "_type": "endpoint",
-          "name": "Create",
-          "request": {
-            "description": "Add a trace to the database",
-            "url": {
-              "raw": "{{baseUrl}}/api/public/traces",
-              "host": [
-                "{{baseUrl}}"
-              ],
-              "path": [
-                "api",
-                "public",
-                "traces"
-              ],
-              "query": [],
-              "variable": []
-            },
-            "header": [],
-            "method": "POST",
-            "auth": null,
-            "body": {
-              "mode": "raw",
-              "raw": "{\n    \"id\": \"example\",\n    \"name\": \"example\",\n    \"userId\": \"example\",\n    \"externalId\": \"example\",\n    \"release\": \"example\",\n    \"version\": \"example\",\n    \"metadata\": \"UNKNOWN\"\n}",
-              "options": {
-                "raw": {
-                  "language": "json"
-                }
-              }
-            }
-          },
-          "response": []
-        },
         {
           "_type": "endpoint",
           "name": "Get",
@@ -780,7 +708,7 @@
           "request": {
             "description": "Get list of traces",
             "url": {
-              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=",
+              "raw": "{{baseUrl}}/api/public/traces?page=&limit=&userId=&name=&orderBy=&tags=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -809,6 +737,16 @@
                   "key": "name",
                   "value": "",
                   "description": null
+                },
+                {
+                  "key": "orderBy",
+                  "value": "",
+                  "description": "Format of the string sort_by=timestamp.asc (id, timestamp, name, userId, release, version, public, bookmarked, sessionId)"
+                },
+                {
+                  "key": "tags",
+                  "value": "",
+                  "description": "Only traces that include all of these tags will be returned."
                 }
               ],
               "variable": []

jest.config.mjs

@@ -18,6 +18,8 @@ const serverTestConfig = {
   testEnvironment: "jest-environment-node",
 };
 
+// To avoid the "Cannot use import statement outside a module" errors while transforming ESM.
+const esModules = ["superjson"];
 // Add any custom config to be passed to Jest
 /** @type {import('jest').Config} */
 const config = {
@@ -26,7 +28,10 @@ const config = {
   verbose: true,
   projects: [
     await createJestConfig(clientTestConfig)(),
-    await createJestConfig(serverTestConfig)(),
+    {
+      ...(await createJestConfig(serverTestConfig)()),
+      transformIgnorePatterns: [`/node_modules/(?!(${esModules.join("|")})/)`],
+    },
   ],
 };
 

next.config.mjs

@@ -23,27 +23,37 @@ const nextConfig = {
   output: "standalone",
 
   async headers() {
-    // Required to check authentication status from langfuse.com
-    if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined)
-      return [
-        {
-          source: "/api/auth/session",
-          headers: [
-            {
-              key: "Access-Control-Allow-Origin",
-              value: "https://langfuse.com",
-            },
-            { key: "Access-Control-Allow-Credentials", value: "true" },
-            { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-            {
-              key: "Access-Control-Allow-Headers",
-              value: "Content-Type, Authorization",
-            },
-          ],
-        },
-      ];
-
-    return [];
+    return [
+      {
+        source: "/:path*",
+        headers: [
+          {
+            key: "x-frame-options",
+            value: "SAMEORIGIN",
+          },
+        ],
+      },
+      // Required to check authentication status from langfuse.com
+      ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined ?
+        [
+          {
+            source: "/api/auth/session",
+            headers: [
+              {
+                key: "Access-Control-Allow-Origin",
+                value: "https://langfuse.com",
+              },
+              { key: "Access-Control-Allow-Credentials", value: "true" },
+              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+              {
+                key: "Access-Control-Allow-Headers",
+                value: "Content-Type, Authorization",
+              },
+            ]
+          },
+        ] : []
+      )
+    ]
   },
 
   // webassembly support for @dqbd/tiktoken

package.json

@@ -1,38 +1,46 @@
 {
   "name": "langfuse-core",
-  "version": "1.8.2",
+  "version": "2.2.0",
   "private": true,
   "scripts": {
     "prebuild": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
     "build": "next build",
     "dev": "next dev",
+    "dx": "npm i && npm run db:reset && npm run db:seed:examples && npm run dev",
     "postinstall": "prisma generate",
     "lint": "next lint",
+    "lint:fix": "next lint --fix",
     "prettier": "prettier --write ./src *.{ts,js}",
     "clean": "rm -rf node_modules",
     "start": "next start",
     "test": "jest --runInBand",
     "test:watch": "jest --watch --runInBand",
     "test:e2e": "playwright test",
-    "db:migrate": "npx prisma migrate dev",
+    "infra:dev:up": "docker-compose  -f docker-compose.dev.yml up -d",
+    "infra:dev:down": "docker-compose -f docker-compose.dev.yml down",
+    "db:migrate": "DISABLE_ERD=false npx prisma migrate dev",
+    "db:reset": "npx prisma migrate reset",
     "db:seed": "npx prisma db seed",
     "db:seed:examples": "npx prisma db seed -- --environment examples",
-    "release": "release-it"
+    "release": "release-it",
+    "models:migrate": "tsx scripts/model-match.ts"
   },
   "prisma": {
     "seed": "ts-node -r tsconfig-paths/register --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
-    "@emotion/react": "^11.11.1",
-    "@emotion/styled": "^11.11.0",
-    "@formkit/auto-animate": "^0.8.1",
-    "@headlessui/react": "^1.7.17",
-    "@heroicons/react": "^2.0.18",
-    "@hookform/resolvers": "^3.3.2",
+    "@aws-sdk/client-s3": "^3.507.0",
+    "@aws-sdk/s3-request-presigner": "^3.507.0",
+    "@headlessui/react": "^1.7.18",
+    "@heroicons/react": "^2.1.1",
+    "@hookform/resolvers": "^3.3.4",
     "@next-auth/prisma-adapter": "^1.0.7",
-    "@prisma/client": "^5.6.0",
+    "@prisma/client": "^5.9.1",
+    "@radix-ui/react-accordion": "^1.1.2",
+    "@radix-ui/react-alert-dialog": "^1.0.5",
     "@radix-ui/react-avatar": "^1.0.4",
+    "@radix-ui/react-checkbox": "^1.0.4",
     "@radix-ui/react-collapsible": "^1.0.3",
     "@radix-ui/react-dialog": "^1.0.5",
     "@radix-ui/react-dropdown-menu": "^2.0.6",
@@ -40,6 +48,7 @@
     "@radix-ui/react-icons": "^1.3.0",
     "@radix-ui/react-label": "^2.0.2",
     "@radix-ui/react-popover": "^1.0.7",
+    "@radix-ui/react-scroll-area": "^1.0.5",
     "@radix-ui/react-select": "^2.0.0",
     "@radix-ui/react-separator": "^1.0.3",
     "@radix-ui/react-slider": "^1.1.2",
@@ -48,87 +57,89 @@
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-toggle": "^1.0.3",
     "@radix-ui/react-tooltip": "^1.0.7",
-    "@sentry/nextjs": "^7.83.0",
-    "@sentry/profiling-node": "^1.2.6",
-    "@t3-oss/env-nextjs": "^0.7.1",
+    "@react-email/components": "^0.0.14",
+    "@react-email/render": "^0.0.12",
+    "@sentry/nextjs": "^7.99.0",
+    "@sentry/profiling-node": "^1.3.5",
+    "@sentry/types": "^7.88.0",
+    "@t3-oss/env-nextjs": "^0.8.0",
     "@tailwindcss/forms": "^0.5.7",
-    "@tailwindcss/line-clamp": "^0.4.4",
     "@tanstack/react-query": "^4.36.1",
-    "@tanstack/react-table": "^8.10.7",
+    "@tanstack/react-table": "^8.11.8",
     "@tremor/react": "^3.11.1",
-    "@trpc/client": "^10.44.1",
-    "@trpc/next": "^10.44.1",
-    "@trpc/react-query": "^10.44.1",
-    "@trpc/server": "^10.44.1",
+    "@trpc/client": "^10.45.0",
+    "@trpc/next": "^10.45.0",
+    "@trpc/react-query": "^10.45.0",
+    "@trpc/server": "^10.45.0",
     "@vercel/edge-config": "^0.4.1",
-    "axios": "^1.6.2",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
-    "clsx": "^2.0.0",
-    "cmdk": "^0.2.0",
+    "clsx": "^2.1.0",
+    "cmdk": "^0.2.1",
+    "core-js": "^3.35.1",
     "cors": "^2.8.5",
-    "date-fns": "^2.30.0",
+    "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
-    "js-yaml": "^4.1.0",
+    "js-tiktoken": "^1.0.10",
     "lodash": "^4.17.21",
-    "lucide-react": "^0.294.0",
-    "next": "^14.0.3",
+    "lucide-react": "^0.323.0",
+    "next": "^14.1.0",
     "next-auth": "^4.24.5",
     "next-query-params": "^5.0.0",
-    "next-swagger-doc": "^0.4.0",
-    "posthog-js": "^1.93.3",
-    "posthog-node": "^3.1.3",
+    "nodemailer": "^6.9.9",
+    "posthog-js": "^1.104.4",
+    "posthog-node": "^3.6.2",
     "react": "18.2.0",
-    "react-day-picker": "^8.9.1",
+    "react-day-picker": "^8.10.0",
     "react-dom": "18.2.0",
-    "react-hook-form": "^7.48.2",
-    "react-icons": "^4.12.0",
+    "react-hook-form": "^7.50.1",
+    "react-icons": "^5.0.1",
     "react-responsive": "^9.0.2",
-    "react18-json-view": "^0.2.7-canary.1",
-    "superagent": "^8.1.2",
+    "react18-json-view": "^0.2.7",
     "superjson": "2.2.1",
-    "tailwind-merge": "^2.0.0",
+    "tailwind-merge": "^2.2.1",
     "tailwindcss-animate": "^1.0.7",
-    "tiktoken": "^1.0.11",
-    "ts-pattern": "^5.0.5",
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.1",
     "zod": "^3.22.4"
   },
   "devDependencies": {
-    "@playwright/test": "^1.40.1",
+    "@jedmao/location": "^3.0.0",
+    "@mermaid-js/mermaid-cli": "^10.7.0",
+    "@playwright/test": "^1.41.2",
     "@release-it/bumper": "^6.0.1",
-    "@testing-library/jest-dom": "^6.1.4",
+    "@testing-library/jest-dom": "^6.4.2",
+    "@testing-library/react": "^14.2.1",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
-    "@types/eslint": "^8.44.7",
-    "@types/jest": "^29.5.10",
-    "@types/js-yaml": "^4.0.9",
+    "@types/eslint": "^8.56.2",
+    "@types/jest": "^29.5.12",
     "@types/lodash": "^4.14.202",
-    "@types/node": "20.10.0",
-    "@types/react": "^18.2.39",
-    "@types/react-dom": "^18.2.17",
-    "@types/swagger-ui-react": "^4.18.3",
-    "@types/uuid": "^9.0.7",
-    "@typescript-eslint/eslint-plugin": "^6.13.1",
-    "@typescript-eslint/parser": "^6.13.1",
-    "autoprefixer": "^10.4.16",
+    "@types/node": "20.10.5",
+    "@types/nodemailer": "^6.4.14",
+    "@types/react": "^18.2.55",
+    "@types/react-dom": "^18.2.18",
+    "@types/uuid": "^9.0.8",
+    "@typescript-eslint/eslint-plugin": "^6.20.0",
+    "@typescript-eslint/parser": "^6.21.0",
+    "autoprefixer": "^10.4.17",
     "dotenv-cli": "^7.3.0",
-    "eslint": "^8.54.0",
-    "eslint-config-next": "^14.0.3",
+    "eslint": "^8.56.0",
+    "eslint-config-next": "^14.1.0",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
-    "postcss": "^8.4.31",
-    "prettier": "^3.1.0",
-    "prettier-plugin-tailwindcss": "^0.5.7",
-    "prisma": "^5.6.0",
-    "release-it": "^17.0.0",
-    "swagger-jsdoc": "^6.2.8",
-    "tailwindcss": "^3.3.5",
-    "ts-node": "^10.9.1",
+    "postcss": "^8.4.34",
+    "prettier": "^3.2.5",
+    "prettier-plugin-tailwindcss": "^0.5.11",
+    "prisma": "^5.9.1",
+    "prisma-erd-generator": "^1.11.2",
+    "release-it": "^17.0.3",
+    "tailwindcss": "^3.4.1",
+    "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.3.2"
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
   },
   "ct3aMetadata": {
     "initVersion": "7.13.0"
@@ -145,7 +156,12 @@
       "release": true,
       "web": true,
       "autoGenerate": true,
-      "releaseName": "v${version}"
+      "releaseName": "v${version}",
+      "comments": {
+        "submit": true,
+        "issue": ":rocket: _This issue has been resolved in v${version}. See [${releaseName}](${releaseUrl}) for release notes._",
+        "pr": ":rocket: _This pull request is included in v${version}. See [${releaseName}](${releaseUrl}) for release notes._"
+      }
     },
     "plugins": {
       "@release-it/bumper": {

prisma/migrations/20231129013314_invites/migration.sql

@@ -0,0 +1,27 @@
+-- CreateTable
+CREATE TABLE "membership_invitations" (
+    "id" TEXT NOT NULL,
+    "email" TEXT NOT NULL,
+    "role" "MembershipRole" NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "sender_id" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "membership_invitations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "membership_invitations_id_key" ON "membership_invitations"("id");
+
+-- CreateIndex
+CREATE INDEX "membership_invitations_project_id_idx" ON "membership_invitations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "membership_invitations_email_idx" ON "membership_invitations"("email");
+
+-- AddForeignKey
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "membership_invitations" ADD CONSTRAINT "membership_invitations_sender_id_fkey" FOREIGN KEY ("sender_id") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;

prisma/migrations/20231130003317_trace_session_input_output/migration.sql

@@ -0,0 +1,34 @@
+-- AlterTable
+ALTER TABLE "traces" ADD COLUMN     "input" JSONB,
+ADD COLUMN     "output" JSONB,
+ADD COLUMN     "session_id" TEXT;
+
+-- CreateTable
+CREATE TABLE "trace_sessions" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "bookmarked" BOOLEAN NOT NULL DEFAULT false,
+    "public" BOOLEAN NOT NULL DEFAULT false,
+
+    CONSTRAINT "trace_sessions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "trace_sessions_project_id_idx" ON "trace_sessions"("project_id");
+
+-- CreateIndex
+CREATE INDEX "trace_sessions_created_at_idx" ON "trace_sessions"("created_at");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "trace_sessions_id_project_id_key" ON "trace_sessions"("id", "project_id");
+
+-- CreateIndex
+CREATE INDEX "traces_session_id_idx" ON "traces"("session_id");
+
+-- AddForeignKey
+ALTER TABLE "trace_sessions" ADD CONSTRAINT "trace_sessions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "traces" ADD CONSTRAINT "traces_session_id_fkey" FOREIGN KEY ("session_id") REFERENCES "trace_sessions"("id") ON DELETE SET NULL ON UPDATE CASCADE;

prisma/migrations/20231204223505_add_unit_to_observations/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+-- Adding this migration does not lock the postgres table. For non volaitile data, the default value is stored in the 
+-- table metadata and accessed on read query time. No table re-write is required (https://www.postgresql.org/docs/current/sql-altertable.html#:~:text=When%20a%20column%20is%20added,is%20specified%2C%20NULL%20is%20used.)
+ALTER TABLE "observations" ADD COLUMN "unit" TEXT NOT NULL DEFAULT 'TOKENS';

prisma/migrations/20231223230007_cloud_config/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN     "cloud_config" JSONB;

prisma/migrations/20231223230008_accounts_add_cols_azure_ad_auth/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "Account" ADD COLUMN     "expires_in" INTEGER,
+ADD COLUMN     "ext_expires_in" INTEGER;

prisma/migrations/20231230151856_add_prompt_table/migration.sql

@@ -0,0 +1,29 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "prompt_id" TEXT;
+
+-- CreateTable
+CREATE TABLE "prompts" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "created_by" TEXT NOT NULL,
+    "prompt" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "version" INTEGER NOT NULL,
+    "is_active" BOOLEAN NOT NULL,
+
+    CONSTRAINT "prompts_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "prompts_project_id_name_version_idx" ON "prompts"("project_id", "name", "version");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "prompts_project_id_name_version_key" ON "prompts"("project_id", "name", "version");
+
+-- AddForeignKey
+ALTER TABLE "observations" ADD CONSTRAINT "observations_prompt_id_fkey" FOREIGN KEY ("prompt_id") REFERENCES "prompts"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "prompts" ADD CONSTRAINT "prompts_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/migrations/20240103135918_add_pricings/migration.sql

@@ -0,0 +1,30 @@
+-- Migration to add gpt-3.5-turbo-1106 and gpt-3.5-turbo-instruct models
+
+INSERT INTO pricings (
+  id,
+  model_name, 
+  pricing_unit, 
+  price,
+  currency,
+  token_type
+)
+VALUES
+  ('clqqpc2pr000008l3hvy63gxy','gpt-3.5-turbo-1106', 'PER_1000_TOKENS', 0.001, 'USD', 'PROMPT'),
+  ('clqqpcb6d000208l3atrfbmou','gpt-3.5-turbo-1106', 'PER_1000_TOKENS', 0.002, 'USD', 'COMPLETION'),
+  ('clqqpdh45000008lfgrnx76cv','gpt-3.5-turbo-instruct', 'PER_1000_TOKENS', 0.0015, 'USD', 'PROMPT'),
+  ('clqqpdjya000108lf3s4b4c4m','gpt-3.5-turbo-instruct', 'PER_1000_TOKENS', 0.002, 'USD', 'COMPLETION'),
+  ('clnnuiuq6000008l4dxp43wc6', 'claude-instant-1.2', 'PER_1000_TOKENS', 0.00163, 'USD', 'PROMPT'),
+  ('clnnujlmj000108l48wne4ii9', 'claude-instant-1.2', 'PER_1000_TOKENS', 0.00551, 'USD', 'COMPLETION'),
+  ('clnnukutg000208l49qqt9lyr', 'claude-instant-1.1', 'PER_1000_TOKENS', 0.00163, 'USD', 'PROMPT'),
+  ('clnnun3x3000408l490qz9uv0', 'claude-instant-1.1', 'PER_1000_TOKENS', 0.00551, 'USD', 'COMPLETION'),
+  ('clnnuosvy000508l4gsjy2pp4', 'claude-2.0', 'PER_1000_TOKENS', 0.01102, 'USD', 'PROMPT'),
+  ('clnnuqcns000608l40qaz8trt', 'claude-2.0', 'PER_1000_TOKENS', 0.03268, 'USD', 'COMPLETION'),
+  ('clnnuuif8000808l4gal4fjq4', 'claude-1.0', 'PER_1000_TOKENS', 0.01102, 'USD', 'PROMPT'),
+  ('clnnutptp000708l47r091vvd', 'claude-1.0', 'PER_1000_TOKENS', 0.03268, 'USD', 'COMPLETION'),
+  ('clnon8riv000308mlgfr1agiv', 'text-embedding-ada-002', 'PER_1000_TOKENS', 0.0001, 'USD', 'PROMPT'),
+  ('clnon9kfz000408ml1bg81o6z', 'text-embedding-ada-002', 'PER_1000_TOKENS', 0.0001, 'USD', 'COMPLETION'),
+  ('clqwniv8a000d08l2frjl7mmw', 'codechat-bison-32k', 'PER_1000_CHARS', 0.0005, 'USD', 'PROMPT'),
+  ('clqwnj044000e08l2dfjm5g90', 'chat-bison-32k', 'PER_1000_CHARS', 0.0005, 'USD', 'PROMPT'),
+  ('clqwnj47r000f08l2a16fekls', 'chat-bison-32k', 'PER_1000_CHARS', 0.0005, 'USD', 'COMPLETION'),
+  ('clqwnj863000g08l2bwxgdapm', 'codechat-bison-32k', 'PER_1000_CHARS', 0.0005, 'USD', 'COMPLETION')
+ON CONFLICT (id) DO NOTHING;

prisma/migrations/20240104210051_add_model_indices/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "observations_model_idx" ON "observations"("model");
+

prisma/migrations/20240104210052_add_model_indices_pricing/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "pricings_model_name_idx" ON "pricings"("model_name");

prisma/migrations/20240105010215_add_tags_in_traces/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "traces" ADD COLUMN     "tags" TEXT[] DEFAULT ARRAY[]::TEXT[];

prisma/migrations/20240105170551_index_tags_in_traces/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "traces_tags_idx" ON "traces" USING GIN ("tags" array_ops);

prisma/migrations/20240106195340_drop_dataset_status/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `status` on the `datasets` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "datasets" DROP COLUMN "status";

prisma/migrations/20240111152124_add_gpt_35_pricing/migration.sql

@@ -0,0 +1,16 @@
+-- Migration to add gpt-35 spelling
+
+INSERT INTO pricings (
+  id,
+  model_name, 
+  pricing_unit, 
+  price,
+  currency,
+  token_type
+)
+VALUES
+  ('clqqpc2pr000008l3hvy63gxy1','gpt-35-turbo-1106', 'PER_1000_TOKENS', 0.001, 'USD', 'PROMPT'),
+  ('clqqpcb6d000208l3atrfbmou1','gpt-35-turbo-1106', 'PER_1000_TOKENS', 0.002, 'USD', 'COMPLETION'),
+  ('clqqpdh45000008lfgrnx76cv1','gpt-35-turbo-instruct', 'PER_1000_TOKENS', 0.0015, 'USD', 'PROMPT'),
+  ('clqqpdjya000108lf3s4b4c4m1','gpt-35-turbo-instruct', 'PER_1000_TOKENS', 0.002, 'USD', 'COMPLETION')
+ON CONFLICT (id) DO NOTHING;

prisma/migrations/20240117151938_traces_remove_unique_id_external/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX "traces_project_id_external_id_key";

prisma/migrations/20240117165747_add_cost_to_observations/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "input_cost" DECIMAL(65,30),
+ADD COLUMN     "output_cost" DECIMAL(65,30),
+ADD COLUMN     "total_cost" DECIMAL(65,30);

prisma/migrations/20240118204639_add_models_table/migration.sql

@@ -0,0 +1,20 @@
+-- CreateTable
+CREATE TABLE "models" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT,
+    "model_name" TEXT NOT NULL,
+    "match_pattern" TEXT NOT NULL,
+    "start_date" TIMESTAMP(3),
+    "input_price" DECIMAL(65,30),
+    "output_price" DECIMAL(65,30),
+    "total_price" DECIMAL(65,30),
+    "unit" TEXT NOT NULL DEFAULT 'TOKENS',
+    "tokenizer_config" JSONB NOT NULL,
+
+    CONSTRAINT "models_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "models" ADD CONSTRAINT "models_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/migrations/20240118204936_add_internal_model/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "observations" ADD COLUMN     "internal_model" TEXT;

prisma/migrations/20240118204937_add_observations_view/migration.sql

@@ -0,0 +1,49 @@
+CREATE VIEW "observations_view" AS
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost"
+FROM
+    observations o
+LEFT JOIN models m ON m.id = (
+    SELECT
+        id
+    FROM
+        models
+    WHERE (project_id = o.project_id OR project_id IS NULL)
+    AND model_name = o.internal_model
+    AND (start_date < o.start_time OR start_date is NULL)
+    AND o.unit::TEXT = unit
+    ORDER BY
+        project_id ASC, -- in postgres, NULLs are sorted first
+        start_date DESC
+    LIMIT 1
+)

prisma/migrations/20240118235424_add_index_to_models/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[project_id,model_name,start_date,unit]` on the table `models` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- CreateIndex
+CREATE INDEX "models_project_id_model_name_idx" ON "models"("project_id", "model_name");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "models_project_id_model_name_start_date_unit_key" ON "models"("project_id", "model_name", "start_date", "unit");

prisma/migrations/20240119140941_add_tokenizer_id/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "models" ADD COLUMN     "tokenizer_id" TEXT;

prisma/migrations/20240119164147_make_model_params_nullable/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "models" ALTER COLUMN "tokenizer_config" DROP NOT NULL;

prisma/migrations/20240119164148_add_models/migration.sql

@@ -0,0 +1,84 @@
+-- This is an empty migration.
+
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  --project_id, model_name, match_pattern, start_date, input_price, output_price, total_price, unit, tokenizer_id, tokenizer_config
+  --https://openai.com/pricing
+  -- GPT-4 Turbo
+  ('clrkvq6iq000008ju6c16gynt', NULL, 'gpt-4-turbo', '(?i)^(gpt-4-1106-preview)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-1106-preview" }'),
+  ('clrkvx5gp000108juaogs54ea', NULL, 'gpt-4-turbo-vision', '(?i)^(gpt-4-vision-preview)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-vision-preview" }'),
+  
+  -- GPT-4
+  ('clrntkjgy000f08jx79v9g1xj', NULL, 'gpt-4', '(?i)^(gpt-4)$', NULL, 0.00003, 0.00006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4" }'),
+  ('clrkwk4cc000908l537kl0rx3', NULL, 'gpt-4-0613', '(?i)^(gpt-4-0613)$', NULL, 0.00003, 0.00006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-0613" }'),
+  ('clrntkjgy000e08jx4x6uawoo', NULL, 'gpt-4-0314', '(?i)^(gpt-4-0314)$', NULL, 0.00003, 0.00006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-0314" }'),
+  
+  ('clrkvyzgw000308jue4hse4j9', NULL, 'gpt-4-32k', '(?i)^(gpt-4-32k)$', NULL, 0.00006, 0.00012, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-32k" }'),
+  ('clrkwk4cb000108l5hwwh3zdi', NULL, 'gpt-4-32k-0613', '(?i)^(gpt-4-32k-0613)$', NULL, 0.00006, 0.00012, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-32k-0613" }'),
+  ('clrntkjgy000d08jx0p4y9h4l', NULL, 'gpt-4-32k-0314', '(?i)^(gpt-4-32k-0314)$', NULL, 0.00006, 0.00012, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4-32k-0314" }'),
+  
+  -- GPT 3
+  
+  ('clrkwk4cc000a08l562uc3s9g', NULL, 'gpt-3.5-turbo-instruct', '(?i)^(gpt-)(35|3.5)(-turbo-instruct)$', NULL, 0.0000015, 0.000002, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo" }'),
+  ('clrkwk4cb000408l576jl7koo', NULL, 'gpt-3.5-turbo', '(?i)^(gpt-)(35|3.5)(-turbo)$', '2023-11-06', 0.000001, 0.000002, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo" }'),
+  ('clrkwk4cb000208l59yvb9yq8', NULL, 'gpt-3.5-turbo-1106', '(?i)^(gpt-)(35|3.5)(-turbo-1106)$', NULL, 0.000001, 0.000002, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-1106" }'),
+
+  ('clrntkjgy000c08jxesb30p3f', NULL, 'gpt-3.5-turbo', '(?i)^(gpt-)(35|3.5)(-turbo)$', '2023-06-27', 0.0000015, 0.000002, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo" }'),
+  ('clrkwk4cc000808l51xmk4uic', NULL, 'gpt-3.5-turbo-0613', '(?i)^(gpt-)(35|3.5)(-turbo-0613)$', NULL, 0.0000015, 0.000002, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-0613" }'),
+
+  ('clrntkjgy000b08jx769q1bah', NULL, 'gpt-3.5-turbo', '(?i)^(gpt-)(35|3.5)(-turbo)$', NULL,  0.000002,  0.000002, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 4, "tokensPerName": -1, "tokenizerModel": "gpt-3.5-turbo" }'),
+  ('clrntkjgy000a08jx4e062mr0', NULL, 'gpt-3.5-turbo-0301', '(?i)^(gpt-)(35|3.5)(-turbo-0301)$', NULL,  0.000002,  0.000002, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 4, "tokensPerName": -1, "tokenizerModel": "gpt-3.5-turbo-0301" }'),
+
+
+  ('clrkwk4cb000308l5go4b6otm', NULL, 'gpt-3.5-turbo-16k', '(?i)^(gpt-)(35|3.5)(-turbo-16k)$', NULL, 0.00003, 0.00004, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-16k" }'),
+  ('clrntjt89000a08jw0gcdbd5a', NULL, 'gpt-3.5-turbo-16k-0613', '(?i)^(gpt-)(35|3.5)(-turbo-16k-0613)$', NULL, 0.00003, 0.00004, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-16k-0613" }'),
+
+
+
+
+  -- nothing earlier required for ada
+  ('clrntjt89000908jwhvkz5crm', NULL, 'text-embedding-ada-002', '(?i)^(text-embedding-ada-002)$', '2022-12-06', NULL, NULL, 0.0000001, 'TOKENS', 'openai', NULL),
+  ('clrntjt89000908jwhvkz5crg', NULL, 'text-embedding-ada-002-v2', '(?i)^(text-embedding-ada-002-v2)$', '2022-12-06', NULL, NULL, 0.0000001, 'TOKENS', 'openai', NULL),
+  
+
+  
+  
+  -- legacy price 2023-08-22 https://platform.openai.com/docs/deprecations/2023-07-06-gpt-and-embeddings
+  ('clrntjt89000108jwcou1af71', NULL, 'text-ada-001', '(?i)^(text-ada-001)$', NULL, NULL, NULL, 0.000004, 'TOKENS', 'openai', NULL),
+  ('clrntjt89000208jwawjr894q', NULL, 'text-babbage-001', '(?i)^(text-babbage-001)$', NULL, NULL, NULL, 0.0000005, 'TOKENS', 'openai', NULL),
+  ('clrp1wopz000708l079w02hkc', NULL, 'text-babbage-002', '(?i)^(text-babbage-002)$', NULL, NULL, NULL, 0.0000005, 'TOKENS', 'openai', NULL),
+  ('clrntjt89000308jw0jtfa4rs', NULL, 'text-curie-001', '(?i)^(text-curie-001)$', NULL, NULL, NULL, 0.00002, 'TOKENS', 'openai', NULL),
+  ('clrntjt89000408jwc2c93h6i', NULL, 'text-davinci-001', '(?i)^(text-davinci-001)$', NULL, NULL, NULL, 0.00002, 'TOKENS', 'openai', NULL),
+  ('clrntjt89000508jw192m64qi', NULL, 'text-davinci-002', '(?i)^(text-davinci-002)$', NULL, NULL, NULL, 0.00002, 'TOKENS', 'openai', NULL),
+  ('clrntjt89000608jw4m3x5s55', NULL, 'text-davinci-003', '(?i)^(text-davinci-003)$', NULL, NULL, NULL, 0.00002, 'TOKENS', 'openai', NULL),
+
+
+  -- claude
+  ('clrnwbota000908jsgg9mb1ml', NULL, 'claude-instant-1', '(?i)^(claude-instant-1)$', NULL, 0.00000163, 0.00000551, NULL, 'CHARACTERS', 'claude', NULL),
+  ('clrnwb41q000308jsfrac9uh6', NULL, 'claude-instant-1.2', '(?i)^(claude-instant-1.2)$', NULL, 0.00000163, 0.00000551, NULL, 'CHARACTERS', 'claude', NULL),
+  ('clrnwbd1m000508js4hxu6o7n', NULL, 'claude-2.1', '(?i)^(claude-2.1)$', NULL, 0.000008, 0.000024, NULL, 'CHARACTERS', 'claude', NULL),
+  ('clrnwb836000408jsallr6u11', NULL, 'claude-2.0', '(?i)^(claude-2.0)$', NULL, 0.000008, 0.000024, NULL, 'CHARACTERS', 'claude', NULL),
+  ('clrnwbg2b000608jse2pp4q2d', NULL, 'claude-1.3', '(?i)^(claude-1.3)$', NULL, 0.000008, 0.000024, NULL, 'CHARACTERS', 'claude', NULL),
+  ('clrnwbi9d000708jseiy44k26', NULL, 'claude-1.2', '(?i)^(claude-1.2)$', NULL, 0.000008, 0.000024, NULL, 'CHARACTERS', 'claude', NULL),
+  ('clrnwblo0000808jsc1385hdp', NULL, 'claude-1.1', '(?i)^(claude-1.1)$', NULL, 0.000008, 0.000024, NULL, 'CHARACTERS', 'claude', NULL),
+
+
+  -- vertex
+  ('clrp1wopz000808l09nwy32xh', NULL, 'codechat-bison-32k', '(?i)^(codechat-bison-32k)$', NULL, 0.0000005, 0.0000025, NULL, 'TOKENS', 'vertex', NULL),
+  ('clrp1wopz000408l05xcycki1', NULL, 'chat-bison-32k', '(?i)^(chat-bison-32k)$', NULL, 0.0000005, 0.0000025, NULL, 'TOKENS', 'vertex', NULL)
+
+
+

prisma/migrations/20240124140443_session_composite_key/migration.sql

@@ -0,0 +1,18 @@
+/*
+  Warnings:
+
+  - The primary key for the `trace_sessions` table will be changed. If it partially fails, the table could be left without primary key constraint.
+
+*/
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_fkey";
+
+-- DropIndex
+DROP INDEX "trace_sessions_id_project_id_key";
+
+-- AlterTable
+ALTER TABLE "trace_sessions" DROP CONSTRAINT "trace_sessions_pkey",
+ADD CONSTRAINT "trace_sessions_pkey" PRIMARY KEY ("id", "project_id");
+
+-- AddForeignKey
+ALTER TABLE "traces" ADD CONSTRAINT "traces_session_id_project_id_fkey" FOREIGN KEY ("session_id", "project_id") REFERENCES "trace_sessions"("id", "project_id") ON DELETE RESTRICT ON UPDATE CASCADE;

prisma/migrations/20240124164148_correct_models/migration.sql

@@ -0,0 +1,42 @@
+-- This is an empty migration.
+
+DELETE FROM models
+WHERE id in ('clrntjt89000908jwhvkz5crm', 'clrntjt89000908jwhvkz5crg', 'clrntjt89000108jwcou1af71', 'clrntjt89000208jwawjr894q', 'clrntjt89000308jw0jtfa4rs', 'clrntjt89000408jwc2c93h6i', 'clrntjt89000508jw192m64qi', 'clrntjt89000608jw4m3x5s55', 'clrp1wopz000708l079w02hkc');
+
+
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- nothing earlier required for ada
+  ('clrntjt89000908jwhvkz5crm', NULL, 'text-embedding-ada-002', '(?i)^(text-embedding-ada-002)$', '2022-12-06', NULL, NULL, 0.0000001, 'TOKENS', 'openai', '{"tokenizerModel": "text-embedding-ada-002"}'),
+  ('clrntjt89000908jwhvkz5crg', NULL, 'text-embedding-ada-002-v2', '(?i)^(text-embedding-ada-002-v2)$', '2022-12-06', NULL, NULL, 0.0000001, 'TOKENS', 'openai', '{"tokenizerModel": "text-embedding-ada-002"}'),
+  
+
+  
+  
+  -- -- legacy price 2023-08-22 https://platform.openai.com/docs/deprecations/2023-07-06-gpt-and-embeddings
+  ('clrntjt89000108jwcou1af71', NULL, 'text-ada-001', '(?i)^(text-ada-001)$', NULL, NULL, NULL, 0.000004, 'TOKENS', 'openai', '{"tokenizerModel": "text-ada-001"}'),
+  ('clrntjt89000208jwawjr894q', NULL, 'text-babbage-001', '(?i)^(text-babbage-001)$', NULL, NULL, NULL, 0.0000005, 'TOKENS', 'openai', '{"tokenizerModel": "text-babbage-001"}'),
+  ('clrntjt89000308jw0jtfa4rs', NULL, 'text-curie-001', '(?i)^(text-curie-001)$', NULL, NULL, NULL, 0.00002, 'TOKENS', 'openai', '{"tokenizerModel": "text-curie-001"}'),
+  ('clrntjt89000408jwc2c93h6i', NULL, 'text-davinci-001', '(?i)^(text-davinci-001)$', NULL, NULL, NULL, 0.00002, 'TOKENS', 'openai', '{"tokenizerModel": "text-davinci-001"}'),
+  ('clrntjt89000508jw192m64qi', NULL, 'text-davinci-002', '(?i)^(text-davinci-002)$', NULL, NULL, NULL, 0.00002, 'TOKENS', 'openai', '{"tokenizerModel": "text-davinci-002"}'),
+  ('clrntjt89000608jw4m3x5s55', NULL, 'text-davinci-003', '(?i)^(text-davinci-003)$', NULL, NULL, NULL, 0.00002, 'TOKENS', 'openai', '{"tokenizerModel": "text-davinci-003"}'),
+
+  ('clrs2dnql000108l46vo0gp2t', NULL, 'babbage-002', '(?i)^(babbage-002)$', NULL, 0.0000004, 0.0000016, 0.0000005, 'TOKENS', 'openai', '{"tokenizerModel": "babbage-002"}'),
+  ('clrs2ds35000208l4g4b0hi3u', NULL, 'davinci-002', '(?i)^(davinci-002)$', NULL, 0.0000060, 0.0000120, 0.0000005, 'TOKENS', 'openai', '{"tokenizerModel": "davinci-002"}')
+
+
+
+

prisma/migrations/20240126184148_new_models copy/migration.sql

@@ -0,0 +1,39 @@
+-- This is an empty migration.
+
+DELETE FROM models
+WHERE id in ('clrp1wopz000808l09nwy32xh', 'clrp1wopz000408l05xcycki1','clrs2dnql000108l46vo0gp2t', 'clrs2ds35000208l4g4b0hi3u');
+
+
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://openai.com/blog/new-embedding-models-and-api-updates
+  -- tokenizers are best guess for embedding models
+  ('clruwn3pc00010al7bl611c8o', NULL, 'text-embedding-3-small', '(?i)^(text-embedding-3-small)$', NULL, NULL, NULL, 0.00000002, 'TOKENS', 'openai', '{"tokenizerModel": "text-embedding-ada-002"}'),
+  ('clruwn76700020al7gp8e4g4l', NULL, 'text-embedding-ada-002-v2', '(?i)^(text-embedding-3-large)$', NULL, NULL, NULL, 0.00000013, 'TOKENS', 'openai', '{"tokenizerModel": "text-embedding-ada-002"}'),
+  
+  ('clruwnahl00030al7ab9rark7', NULL, 'gpt-3.5-turbo-0125', '(?i)^(gpt-)(35|3.5)(-turbo-0125)$', NULL, 0.0000005, 0.0000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo" }'),
+  ('clruwnahl00040al78f1lb0at', NULL, 'gpt-3.5-turbo', '(?i)^(gpt-)(35|3.5)(-turbo)$', '2024-02-08', 0.0000005, 0.0000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo" }'),
+
+  ('clruwnahl00050al796ck3p44', NULL, 'gpt-4-0125-preview', '(?i)^(gpt-4-0125-preview)$', NULL, 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4" }'),
+  ('clruwnahl00060al74fcfehas', NULL, 'gpt-4-turbo-preview', '(?i)^(gpt-4-turbo-preview)$', NULL, 0.00003, 0.00006, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4" }'),
+
+  -- fix tokenizer for vertx
+  ('clrp1wopz000808l09nwy32xh', NULL, 'codechat-bison-32k', '(?i)^(codechat-bison-32k)$', NULL, 0.0000005, 0.0000025, NULL, 'TOKENS', NULL, NULL),
+  ('clrp1wopz000408l05xcycki1', NULL, 'chat-bison-32k', '(?i)^(chat-bison-32k)$', NULL, 0.0000005, 0.0000025, NULL, 'TOKENS', NULL, NULL),
+
+  -- fix prices
+  ('clrs2dnql000108l46vo0gp2t', NULL, 'babbage-002', '(?i)^(babbage-002)$', NULL, 0.0000004, 0.0000016, NULL, 'TOKENS', 'openai', '{"tokenizerModel": "babbage-002"}'),
+  ('clrs2ds35000208l4g4b0hi3u', NULL, 'davinci-002', '(?i)^(davinci-002)$', NULL, 0.0000060, 0.0000120, NULL, 'TOKENS', 'openai', '{"tokenizerModel": "davinci-002"}')

prisma/migrations/20240130100110_usage_unit_nullable/migration.sql

@@ -0,0 +1,6 @@
+-- AlterTable
+ALTER TABLE "models" ALTER COLUMN "unit" DROP DEFAULT;
+
+-- AlterTable
+ALTER TABLE "observations" ALTER COLUMN "unit" DROP NOT NULL,
+ALTER COLUMN "unit" DROP DEFAULT;

prisma/migrations/20240130160110_claude_models/migration.sql

@@ -0,0 +1,40 @@
+-- This is an empty migration.
+
+DELETE FROM models
+WHERE id in (
+  'clrnwbota000908jsgg9mb1ml',
+  'clrnwb41q000308jsfrac9uh6', 
+  'clrnwbd1m000508js4hxu6o7n', 
+  'clrnwb836000408jsallr6u11', 
+  'clrnwbg2b000608jse2pp4q2d', 
+  'clrnwbi9d000708jseiy44k26',
+  'clrnwblo0000808jsc1385hdp'
+);
+
+
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- nothing earlier required for ada
+  ('clrnwbota000908jsgg9mb1ml', NULL, 'claude-instant-1', '(?i)^(claude-instant-1)$', NULL, 0.00000163, 0.00000551, NULL, 'TOKENS', 'claude', NULL),
+  ('clrnwb41q000308jsfrac9uh6', NULL, 'claude-instant-1.2', '(?i)^(claude-instant-1.2)$', NULL, 0.00000163, 0.00000551, NULL, 'TOKENS', 'claude', NULL),
+  ('clrnwbd1m000508js4hxu6o7n', NULL, 'claude-2.1', '(?i)^(claude-2.1)$', NULL, 0.000008, 0.000024, NULL, 'TOKENS', 'claude', NULL),
+  ('clrnwb836000408jsallr6u11', NULL, 'claude-2.0', '(?i)^(claude-2.0)$', NULL, 0.000008, 0.000024, NULL, 'TOKENS', 'claude', NULL),
+  ('clrnwbg2b000608jse2pp4q2d', NULL, 'claude-1.3', '(?i)^(claude-1.3)$', NULL, 0.000008, 0.000024, NULL, 'TOKENS', 'claude', NULL),
+  ('clrnwbi9d000708jseiy44k26', NULL, 'claude-1.2', '(?i)^(claude-1.2)$', NULL, 0.000008, 0.000024, NULL, 'TOKENS', 'claude', NULL),
+  ('clrnwblo0000808jsc1385hdp', NULL, 'claude-1.1', '(?i)^(claude-1.1)$', NULL, 0.000008, 0.000024, NULL, 'TOKENS', 'claude', NULL)
+
+
+

prisma/migrations/20240131184148_add_finetuned_and_vertex_models/migration.sql

@@ -0,0 +1,40 @@
+DELETE FROM models
+WHERE id in ('clrp1wopz000808l09nwy32xh', 'clrp1wopz000408l05xcycki1');
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://openai.com/blog/new-embedding-models-and-api-updates
+  -- https://openai.com/blog/gpt-3-5-turbo-fine-tuning-and-api-updates
+  -- ft model tokens are getting counted like the base model https://github.com/openai/tiktoken/blob/db5bda9fc93b3171db6c4afea329394e6b6d31ca/tiktoken/model.py
+  
+  ('cls08r8sq000308jq14ae96f0', NULL, 'ft:gpt-3.5-turbo-1106', '(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$', NULL, 0.000003, 0.000006, NULL, 'TOKENS', 'openai', '{"tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-1106", "tokensPerMessage": 3}'),
+  ('cls08rp99000408jqepxoakjv', NULL, 'ft:gpt-3.5-turbo-0613', '(?i)^(ft:)(gpt-3.5-turbo-0613:)(.+)(:)(.*)(:)(.+)$', NULL, 0.000012, 0.000016, NULL, 'TOKENS', 'openai', '{"tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-0613", "tokensPerMessage": 3}'),
+  ('cls08rv9g000508jq5p4z4nlr', NULL, 'ft:davinci-002', '(?i)^(ft:)(davinci-002:)(.+)(:)(.*)(:)(.+)$$', NULL, 0.000012, 0.000012, NULL, 'TOKENS', 'openai', '{"tokenizerModel": "davinci-002"}'),
+  ('cls08s2bw000608jq57wj4un2', NULL, 'ft:babbage-002', '(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$$', NULL, 0.0000016, 0.0000016, NULL, 'TOKENS', 'openai', '{"tokenizerModel": "babbage-002"}'),
+
+  -- https://cloud.google.com/vertex-ai/docs/generative-ai/pricing
+  ('cls0k4lqt000008ky1o1s8wd5', NULL, 'gemini-pro', '(?i)^(gemini-pro)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls0jni4t000008jk3kyy803r', NULL, 'chat-bison-32k', '(?i)^(chat-bison-32k)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls0iv12d000108l251gf3038', NULL, 'chat-bison', '(?i)^(chat-bison)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls0jmjt3000108l83ix86w0d', NULL, 'text-bison-32k', '(?i)^(text-bison-32k)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls0juygp000308jk2a6x9my2', NULL, 'text-bison', '(?i)^(text-bison)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls0jungb000208jk12gm4gk1', NULL, 'text-unicorn', '(?i)^(text-unicorn)(@[a-zA-Z0-9]+)?$', NULL, 0.0000025, 0.0000075, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls1nyj5q000208l33ne901d8', NULL, 'textembedding-gecko', '(?i)^(textembedding-gecko)(@[a-zA-Z0-9]+)?$', NULL, NULL, NULL, 0.0000001, 'CHARACTERS', NULL, NULL),
+  ('cls1nyyjp000308l31gxy1bih', NULL, 'textembedding-gecko-multilingual', '(?i)^(textembedding-gecko-multilingual)(@[a-zA-Z0-9]+)?$', NULL, NULL, NULL, 0.0000001, 'CHARACTERS', NULL, NULL),
+  ('cls1nzjt3000508l3dnwad3g0', NULL, 'code-gecko', '(?i)^(code-gecko)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls1nzwx4000608l38va7e4tv', NULL, 'code-bison', '(?i)^(code-bison)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls1o053j000708l39f8g4bgs', NULL, 'code-bison-32k', '(?i)^(code-bison-32k)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls0j33v1000008joagkc4lql', NULL, 'codechat-bison-32k', '(?i)^(codechat-bison-32k)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL),
+  ('cls0jmc9v000008l8ee6r3gsd', NULL, 'codechat-bison', '(?i)^(codechat-bison)(@[a-zA-Z0-9]+)?$', NULL, 0.00000025, 0.0000005, NULL, 'CHARACTERS', NULL, NULL)

prisma/migrations/20240203184148_update_pricing/migration.sql

@@ -0,0 +1,27 @@
+-- This is an empty migration.
+
+DELETE FROM models
+WHERE id in ('clrkwk4cb000308l5go4b6otm', 'clrntjt89000a08jw0gcdbd5a');
+
+
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://openai.com/blog/new-embedding-models-and-api-updates
+  
+
+  -- fix prices
+  ('clrkwk4cb000308l5go4b6otm', NULL, 'gpt-3.5-turbo-16k', '(?i)^(gpt-)(35|3.5)(-turbo-16k)$', NULL, 0.000003, 0.000004, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-16k" }'),
+  ('clrntjt89000a08jw0gcdbd5a', NULL, 'gpt-3.5-turbo-16k-0613', '(?i)^(gpt-)(35|3.5)(-turbo-16k-0613)$', NULL, 0.000003, 0.000004, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-16k-0613" }')

prisma/schema.prisma

@@ -2,341 +2,493 @@
 // learn more about it in the docs: https://pris.ly/d/prisma-schema
 
 generator client {
-  provider        = "prisma-client-js"
-  previewFeatures = ["tracing"]
+    provider        = "prisma-client-js"
+    previewFeatures = ["tracing", "views"]
 }
 
 datasource db {
-  provider  = "postgresql"
-  url       = env("DATABASE_URL")
-  directUrl = env("DIRECT_URL")
+    provider          = "postgresql"
+    url               = env("DATABASE_URL")
+    directUrl         = env("DIRECT_URL")
+    shadowDatabaseUrl = env("SHADOW_DATABASE_URL")
+}
+
+generator erd {
+    provider     = "prisma-erd-generator"
+    ignoreTables = ["_prisma_migrations", "Session", "Account", "Example"]
+    disabled     = true
+    ignoreEnums  = true
+    output       = "database.svg"
 }
 
 model Example {
-  id        String   @id @default(cuid())
-  createdAt DateTime @default(now()) @map("created_at")
-  updatedAt DateTime @updatedAt @map("updated_at")
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @updatedAt @map("updated_at")
 }
 
 // Necessary for Next auth
 model Account {
-  id                String  @id @default(cuid())
-  userId            String  @map("user_id")
-  type              String
-  provider          String
-  providerAccountId String
-  refresh_token     String? // @db.Text
-  access_token      String? // @db.Text
-  expires_at        Int?
-  token_type        String?
-  scope             String?
-  id_token          String? // @db.Text
-  session_state     String?
-  user              User    @relation(fields: [userId], references: [id], onDelete: Cascade)
+    id                String  @id @default(cuid())
+    userId            String  @map("user_id")
+    type              String
+    provider          String
+    providerAccountId String
+    refresh_token     String? // @db.Text
+    access_token      String? // @db.Text
+    expires_at        Int?
+    expires_in        Int?
+    ext_expires_in    Int?
+    token_type        String?
+    scope             String?
+    id_token          String? // @db.Text
+    session_state     String?
+    user              User    @relation(fields: [userId], references: [id], onDelete: Cascade)
 
-  @@unique([provider, providerAccountId])
-  @@index([userId])
+    @@unique([provider, providerAccountId])
+    @@index([userId])
 }
 
 model Session {
-  id           String   @id @default(cuid())
-  sessionToken String   @unique @map("session_token")
-  userId       String   @map("user_id")
-  expires      DateTime
-  user         User     @relation(fields: [userId], references: [id], onDelete: Cascade)
+    id           String   @id @default(cuid())
+    sessionToken String   @unique @map("session_token")
+    userId       String   @map("user_id")
+    expires      DateTime
+    user         User     @relation(fields: [userId], references: [id], onDelete: Cascade)
 }
 
 model User {
-  id            String       @id @default(cuid())
-  name          String?
-  email         String?      @unique
-  emailVerified DateTime?    @map("email_verified")
-  password      String?
-  image         String?
-  admin         Boolean      @default(false)
-  accounts      Account[]
-  sessions      Session[]
-  memberships   Membership[]
-  createdAt     DateTime     @default(now()) @map("created_at")
-  updatedAt     DateTime     @default(now()) @updatedAt @map("updated_at")
-  featureFlags  String[]     @default([]) @map("feature_flags")
+    id            String                 @id @default(cuid())
+    name          String?
+    email         String?                @unique
+    emailVerified DateTime?              @map("email_verified")
+    password      String?
+    image         String?
+    admin         Boolean                @default(false)
+    accounts      Account[]
+    sessions      Session[]
+    memberships   Membership[]
+    invitations   MembershipInvitation[]
+    createdAt     DateTime               @default(now()) @map("created_at")
+    updatedAt     DateTime               @default(now()) @updatedAt @map("updated_at")
+    featureFlags  String[]               @default([]) @map("feature_flags")
 
-  @@map("users")
+    @@map("users")
 }
 
 model VerificationToken {
-  identifier String
-  token      String   @unique
-  expires    DateTime
+    identifier String
+    token      String   @unique
+    expires    DateTime
 
-  @@unique([identifier, token])
-  @@map("verification_tokens")
+    @@unique([identifier, token])
+    @@map("verification_tokens")
 }
 
 model Project {
-  id           String        @id @default(cuid())
-  createdAt    DateTime      @default(now()) @map("created_at")
-  updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
-  name         String
-  members      Membership[]
-  traces       Trace[]
-  observations Observation[]
-  apiKeys      ApiKey[]
-  dataset      Dataset[]
-  RawEvents    Events[]
+    id           String                 @id @default(cuid())
+    createdAt    DateTime               @default(now()) @map("created_at")
+    updatedAt    DateTime               @default(now()) @updatedAt @map("updated_at")
+    name         String
+    cloudConfig  Json?                  @map("cloud_config") // Langfuse Cloud, for zod schema see projectsRouter.ts
+    members      Membership[]
+    traces       Trace[]
+    observations Observation[]
+    apiKeys      ApiKey[]
+    dataset      Dataset[]
+    RawEvents    Events[]
+    invitations  MembershipInvitation[]
+    sessions     TraceSession[]
+    Prompt       Prompt[]
+    Model        Model[]
 
-  @@map("projects")
+    @@map("projects")
 }
 
 model ApiKey {
-  id                  String    @id @unique @default(cuid())
-  createdAt           DateTime  @default(now()) @map("created_at")
-  note                String?
-  publicKey           String    @unique @map("public_key")
-  hashedSecretKey     String    @unique @map("hashed_secret_key")
-  fastHashedSecretKey String?   @unique @map("fast_hashed_secret_key")
-  displaySecretKey    String    @map("display_secret_key")
-  lastUsedAt          DateTime? @map("last_used_at")
-  expiresAt           DateTime? @map("expires_at")
-  projectId           String    @map("project_id")
-  project             Project   @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    id                  String    @id @unique @default(cuid())
+    createdAt           DateTime  @default(now()) @map("created_at")
+    note                String?
+    publicKey           String    @unique @map("public_key")
+    hashedSecretKey     String    @unique @map("hashed_secret_key")
+    fastHashedSecretKey String?   @unique @map("fast_hashed_secret_key")
+    displaySecretKey    String    @map("display_secret_key")
+    lastUsedAt          DateTime? @map("last_used_at")
+    expiresAt           DateTime? @map("expires_at")
+    projectId           String    @map("project_id")
+    project             Project   @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
-  @@index(projectId)
-  @@index(publicKey)
-  @@index(hashedSecretKey)
-  @@index(fastHashedSecretKey)
-  @@map("api_keys")
+    @@index(projectId)
+    @@index(publicKey)
+    @@index(hashedSecretKey)
+    @@index(fastHashedSecretKey)
+    @@map("api_keys")
 }
 
 model Membership {
-  projectId String         @map("project_id")
-  project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  userId    String         @map("user_id")
-  user      User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-  role      MembershipRole
-  createdAt DateTime       @default(now()) @map("created_at")
-  updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+    projectId String         @map("project_id")
+    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userId    String         @map("user_id")
+    user      User           @relation(fields: [userId], references: [id], onDelete: Cascade)
+    role      MembershipRole
+    createdAt DateTime       @default(now()) @map("created_at")
+    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
 
-  @@id([projectId, userId])
-  @@index([userId])
-  @@map("memberships")
+    @@id([projectId, userId])
+    @@index([userId])
+    @@map("memberships")
+}
+
+model MembershipInvitation {
+    id        String         @id @unique @default(cuid())
+    email     String
+    role      MembershipRole
+    projectId String         @map("project_id")
+    project   Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    senderId  String?        @map("sender_id")
+    sender    User?          @relation(fields: [senderId], references: [id], onDelete: SetNull)
+    createdAt DateTime       @default(now()) @map("created_at")
+    updatedAt DateTime       @default(now()) @updatedAt @map("updated_at")
+
+    @@index([projectId])
+    @@index([email])
+    @@map("membership_invitations")
 }
 
 enum MembershipRole {
-  OWNER
-  ADMIN
-  MEMBER
-  VIEWER
+    OWNER
+    ADMIN
+    MEMBER
+    VIEWER
+}
+
+model TraceSession {
+    id         String   @default(cuid())
+    createdAt  DateTime @default(now()) @map("created_at")
+    updatedAt  DateTime @default(now()) @updatedAt @map("updated_at")
+    projectId  String   @map("project_id")
+    project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    bookmarked Boolean  @default(false)
+    public     Boolean  @default(false)
+    traces     Trace[]
+
+    @@id([id, projectId])
+    @@index([projectId])
+    @@index([createdAt])
+    @@map("trace_sessions")
 }
 
 model Trace {
-  id         String   @id @default(cuid())
-  externalId String?  @map("external_id")
-  timestamp  DateTime @default(now())
-  name       String?
-  userId     String?  @map("user_id")
-  metadata   Json?
-  release    String?
-  version    String?
-  projectId  String   @map("project_id")
-  project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  public     Boolean  @default(false)
-  bookmarked Boolean  @default(false)
+    id         String        @id @default(cuid())
+    externalId String?       @map("external_id")
+    timestamp  DateTime      @default(now())
+    name       String?
+    userId     String?       @map("user_id")
+    metadata   Json?
+    release    String?
+    version    String?
+    projectId  String        @map("project_id")
+    project    Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    public     Boolean       @default(false)
+    bookmarked Boolean       @default(false)
+    tags       String[]      @default([])
+    input      Json?
+    output     Json?
+    sessionId  String?       @map("session_id")
+    session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
 
-  scores Score[]
+    scores Score[]
 
-  @@unique([projectId, externalId])
-  @@index([projectId])
-  @@index([name])
-  @@index([userId])
-  @@index([externalId])
-  @@index(timestamp)
-  @@index(release)
-  @@map("traces")
+    @@index([projectId])
+    @@index([sessionId])
+    @@index([name])
+    @@index([userId])
+    @@index([externalId])
+    @@index(timestamp)
+    @@index(release)
+    @@index([tags(ops: ArrayOps)], type: Gin)
+    @@map("traces")
 }
 
 model Observation {
-  id                  String           @id @default(cuid())
-  traceId             String?          @map("trace_id")
-  projectId           String           @map("project_id")
-  type                ObservationType
-  startTime           DateTime         @default(now()) @map("start_time")
-  endTime             DateTime?        @map("end_time")
-  name                String?
-  metadata            Json?
-  parentObservationId String?          @map("parent_observation_id")
-  level               ObservationLevel @default(DEFAULT)
-  statusMessage       String?          @map("status_message")
-  version             String?
-  createdAt           DateTime         @default(now()) @map("created_at")
+    id                  String           @id @default(cuid())
+    traceId             String?          @map("trace_id")
+    projectId           String           @map("project_id")
+    type                ObservationType
+    startTime           DateTime         @default(now()) @map("start_time")
+    endTime             DateTime?        @map("end_time")
+    name                String?
+    metadata            Json?
+    parentObservationId String?          @map("parent_observation_id")
+    level               ObservationLevel @default(DEFAULT)
+    statusMessage       String?          @map("status_message")
+    version             String?
+    createdAt           DateTime         @default(now()) @map("created_at")
 
-  // GENERATION ONLY
-  model               String?
-  modelParameters     Json?
-  input               Json?
-  output              Json?
-  promptTokens        Int               @default(0) @map("prompt_tokens")
-  completionTokens    Int               @default(0) @map("completion_tokens")
-  totalTokens         Int               @default(0) @map("total_tokens")
-  completionStartTime DateTime?         @map("completion_start_time")
-  scores              Score[]
-  project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  derivedDatasetItems DatasetItem[]
-  datasetRunItems     DatasetRunItems[]
+    // GENERATION ONLY
+    model               String?
+    internalModel       String?           @map("internal_model")
+    modelParameters     Json?
+    input               Json?
+    output              Json?
+    promptTokens        Int               @default(0) @map("prompt_tokens")
+    completionTokens    Int               @default(0) @map("completion_tokens")
+    totalTokens         Int               @default(0) @map("total_tokens")
+    unit                String?
+    inputCost           Decimal?          @map("input_cost")
+    outputCost          Decimal?          @map("output_cost")
+    totalCost           Decimal?          @map("total_cost")
+    completionStartTime DateTime?         @map("completion_start_time")
+    scores              Score[]
+    project             Project           @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    derivedDatasetItems DatasetItem[]
+    datasetRunItems     DatasetRunItems[]
 
-  @@unique([id, projectId])
-  @@index([traceId])
-  @@index([type])
-  @@index(startTime)
-  @@index(createdAt)
-  @@index(projectId)
-  @@index(parentObservationId)
-  @@map("observations")
+    promptId String? @map("prompt_id")
+    prompt   Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
+
+    @@unique([id, projectId])
+    @@index([traceId])
+    @@index([type])
+    @@index(startTime)
+    @@index(createdAt)
+    @@index(projectId)
+    @@index(parentObservationId)
+    @@index(model)
+    @@map("observations")
 }
 
 enum ObservationType {
-  SPAN
-  EVENT
-  GENERATION
+    SPAN
+    EVENT
+    GENERATION
 }
 
 enum ObservationLevel {
-  DEBUG
-  DEFAULT
-  WARNING
-  ERROR
+    DEBUG
+    DEFAULT
+    WARNING
+    ERROR
 }
 
 model Score {
-  id            String       @id @default(cuid())
-  timestamp     DateTime     @default(now())
-  name          String
-  value         Float
-  comment       String?
-  traceId       String       @map("trace_id")
-  trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
-  observationId String?      @map("observation_id")
-  observation   Observation? @relation(fields: [observationId], references: [id], onDelete: SetNull)
+    id            String       @id @default(cuid())
+    timestamp     DateTime     @default(now())
+    name          String
+    value         Float
+    comment       String?
+    traceId       String       @map("trace_id")
+    trace         Trace        @relation(fields: [traceId], references: [id], onDelete: Cascade)
+    observationId String?      @map("observation_id")
+    observation   Observation? @relation(fields: [observationId], references: [id], onDelete: SetNull)
 
-  @@unique([id, traceId]) // used for upsert
-  @@index([value])
-  @@index([traceId], type: Hash)
-  @@index([observationId], type: Hash)
-  @@map("scores")
+    @@unique([id, traceId]) // used for upsert
+    @@index([value])
+    @@index([traceId], type: Hash)
+    @@index([observationId], type: Hash)
+    @@map("scores")
 }
 
 enum PricingUnit {
-  PER_1000_TOKENS
-  PER_1000_CHARS
+    PER_1000_TOKENS
+    PER_1000_CHARS
 }
 
 enum TokenType {
-  PROMPT
-  COMPLETION
-  TOTAL
+    PROMPT
+    COMPLETION
+    TOTAL
 }
 
 model Pricing {
-  id          String      @id @default(cuid())
-  modelName   String      @map("model_name")
-  pricingUnit PricingUnit @default(PER_1000_TOKENS) @map("pricing_unit")
-  price       Decimal
-  currency    String      @default("USD")
-  tokenType   TokenType   @map("token_type")
+    id          String      @id @default(cuid())
+    modelName   String      @map("model_name")
+    pricingUnit PricingUnit @default(PER_1000_TOKENS) @map("pricing_unit")
+    price       Decimal
+    currency    String      @default("USD")
+    tokenType   TokenType   @map("token_type")
 
-  @@map("pricings")
+    @@index(modelName)
+    @@map("pricings")
 }
 
 model CronJobs {
-  name         String    @id
-  lastRun      DateTime? @map("last_run")
-  jobStartedAt DateTime? @map("job_started_at")
-  state        String?
+    name         String    @id
+    lastRun      DateTime? @map("last_run")
+    jobStartedAt DateTime? @map("job_started_at")
+    state        String?
 
-  @@map("cron_jobs")
+    @@map("cron_jobs")
 }
 
 model Dataset {
-  id           String        @id @default(cuid())
-  name         String
-  status       DatasetStatus @default(ACTIVE)
-  projectId    String        @map("project_id")
-  project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt    DateTime      @default(now()) @map("created_at")
-  updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
-  datasetItems DatasetItem[]
-  datasetRuns  DatasetRuns[]
+    id           String        @id @default(cuid())
+    name         String
+    projectId    String        @map("project_id")
+    project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    createdAt    DateTime      @default(now()) @map("created_at")
+    updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
+    datasetItems DatasetItem[]
+    datasetRuns  DatasetRuns[]
 
-  @@unique([projectId, name])
-  @@index([projectId], type: Hash)
-  @@map("datasets")
+    @@unique([projectId, name])
+    @@index([projectId], type: Hash)
+    @@map("datasets")
 }
 
 model DatasetItem {
-  id                  String            @id @default(cuid())
-  status              DatasetStatus     @default(ACTIVE)
-  input               Json
-  expectedOutput      Json?             @map("expected_output")
-  sourceObservationId String?           @map("source_observation_id")
-  sourceObservation   Observation?      @relation(fields: [sourceObservationId], references: [id], onDelete: SetNull)
-  datasetId           String            @map("dataset_id")
-  dataset             Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
-  createdAt           DateTime          @default(now()) @map("created_at")
-  updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
-  datasetRunItems     DatasetRunItems[]
+    id                  String            @id @default(cuid())
+    status              DatasetStatus     @default(ACTIVE)
+    input               Json
+    expectedOutput      Json?             @map("expected_output")
+    sourceObservationId String?           @map("source_observation_id")
+    sourceObservation   Observation?      @relation(fields: [sourceObservationId], references: [id], onDelete: SetNull)
+    datasetId           String            @map("dataset_id")
+    dataset             Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    createdAt           DateTime          @default(now()) @map("created_at")
+    updatedAt           DateTime          @default(now()) @updatedAt @map("updated_at")
+    datasetRunItems     DatasetRunItems[]
 
-  @@index([sourceObservationId], type: Hash)
-  @@index([datasetId], type: Hash)
-  @@map("dataset_items")
+    @@index([sourceObservationId], type: Hash)
+    @@index([datasetId], type: Hash)
+    @@map("dataset_items")
 }
 
 enum DatasetStatus {
-  ACTIVE
-  ARCHIVED
+    ACTIVE
+    ARCHIVED
 }
 
 model DatasetRuns {
-  id              String            @id @default(cuid())
-  name            String
-  datasetId       String            @map("dataset_id")
-  dataset         Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
-  createdAt       DateTime          @default(now()) @map("created_at")
-  updatedAt       DateTime          @default(now()) @updatedAt @map("updated_at")
-  datasetRunItems DatasetRunItems[]
+    id              String            @id @default(cuid())
+    name            String
+    datasetId       String            @map("dataset_id")
+    dataset         Dataset           @relation(fields: [datasetId], references: [id], onDelete: Cascade)
+    createdAt       DateTime          @default(now()) @map("created_at")
+    updatedAt       DateTime          @default(now()) @updatedAt @map("updated_at")
+    datasetRunItems DatasetRunItems[]
 
-  @@unique([datasetId, name])
-  @@index([datasetId], type: Hash)
-  @@map("dataset_runs")
+    @@unique([datasetId, name])
+    @@index([datasetId], type: Hash)
+    @@map("dataset_runs")
 }
 
 model DatasetRunItems {
-  id            String      @id @default(cuid())
-  datasetRunId  String      @map("dataset_run_id")
-  datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
-  datasetItemId String      @map("dataset_item_id")
-  datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
-  observationId String      @map("observation_id")
-  observation   Observation @relation(fields: [observationId], references: [id], onDelete: Cascade)
-  createdAt     DateTime    @default(now()) @map("created_at")
-  updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
+    id            String      @id @default(cuid())
+    datasetRunId  String      @map("dataset_run_id")
+    datasetRun    DatasetRuns @relation(fields: [datasetRunId], references: [id], onDelete: Cascade)
+    datasetItemId String      @map("dataset_item_id")
+    datasetItem   DatasetItem @relation(fields: [datasetItemId], references: [id], onDelete: Cascade)
+    observationId String      @map("observation_id")
+    observation   Observation @relation(fields: [observationId], references: [id], onDelete: Cascade)
+    createdAt     DateTime    @default(now()) @map("created_at")
+    updatedAt     DateTime    @default(now()) @updatedAt @map("updated_at")
 
-  @@index([datasetRunId], type: Hash)
-  @@index([datasetItemId], type: Hash)
-  @@index([observationId], type: Hash)
-  @@map("dataset_run_items")
+    @@index([datasetRunId], type: Hash)
+    @@index([datasetItemId], type: Hash)
+    @@index([observationId], type: Hash)
+    @@map("dataset_run_items")
 }
 
 model Events {
-  id        String   @id @default(cuid())
-  createdAt DateTime @default(now()) @map("created_at")
-  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
-  projectId String   @map("project_id")
-  project   Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  data      Json
-  headers   Json     @default("{}")
-  url       String?
-  method    String?
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+    projectId String   @map("project_id")
+    project   Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    data      Json
+    headers   Json     @default("{}")
+    url       String?
+    method    String?
 
-  @@index(projectId)
-  @@map("events")
+    @@index(projectId)
+    @@map("events")
+}
+
+model Prompt {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String  @map("project_id")
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    createdBy String @map("created_by")
+
+    prompt      String
+    name        String
+    version     Int
+    isActive    Boolean       @map("is_active")
+    Observation Observation[]
+
+    @@unique([projectId, name, version])
+    @@index([projectId, name, version])
+    @@map("prompts")
+}
+
+model Model {
+    id        String   @id @default(cuid())
+    createdAt DateTime @default(now()) @map("created_at")
+    updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+    projectId String?  @map("project_id")
+    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+    modelName       String    @map("model_name")
+    matchPattern    String    @map("match_pattern")
+    startDate       DateTime? @map("start_date")
+    inputPrice      Decimal?  @map("input_price")
+    outputPrice     Decimal?  @map("output_price")
+    totalPrice      Decimal?  @map("total_price")
+    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, or IMAGES
+    tokenizerId     String?   @map("tokenizer_id")
+    tokenizerConfig Json?     @map("tokenizer_config")
+
+    @@unique([projectId, modelName, startDate, unit])
+    @@index([projectId, modelName])
+    @@map("models")
+}
+
+// This view is a mix of the observation and model. Once prisma supports
+// inheritance, we should remove code duplication here.
+view ObservationView {
+    id                  String           @id @default(cuid())
+    traceId             String?          @map("trace_id")
+    projectId           String           @map("project_id")
+    type                ObservationType
+    startTime           DateTime         @default(now()) @map("start_time")
+    endTime             DateTime?        @map("end_time")
+    name                String?
+    metadata            Json?
+    parentObservationId String?          @map("parent_observation_id")
+    level               ObservationLevel @default(DEFAULT)
+    statusMessage       String?          @map("status_message")
+    version             String?
+    createdAt           DateTime         @default(now()) @map("created_at")
+
+    // GENERATION ONLY
+    model               String?
+    modelParameters     Json?
+    input               Json?
+    output              Json?
+    promptTokens        Int       @default(0) @map("prompt_tokens")
+    completionTokens    Int       @default(0) @map("completion_tokens")
+    totalTokens         Int       @default(0) @map("total_tokens")
+    unit                String?
+    completionStartTime DateTime? @map("completion_start_time")
+
+    promptId String? @map("prompt_id")
+
+    // Model
+    modelId     String?  @map("model_id")
+    inputPrice  Decimal? @map("input_price")
+    outputPrice Decimal? @map("output_price")
+    totalPrice  Decimal? @map("total_price")
+
+    // calculated fields
+    calculatedInputCost  Decimal? @map("calculated_input_cost")
+    calculatedOutputCost Decimal? @map("calculated_output_cost")
+    calculatedTotalCost  Decimal? @map("calculated_total_cost")
+
+    @@map("observations_view")
 }

prisma/seed.ts

@@ -5,6 +5,7 @@ import {
 } from "@/src/features/public-api/lib/apiKeys";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
+import { ModelUsageUnit } from "@/src/constants";
 
 const options = {
   environment: { type: "string" },
@@ -53,6 +54,25 @@ async function main() {
     },
   });
 
+  const prompt = await prisma.prompt.upsert({
+    where: {
+      projectId_name_version: {
+        projectId: seedProjectId,
+        name: "summary-prompt",
+        version: 1,
+      },
+    },
+    create: {
+      name: "summary-prompt",
+      project: { connect: { id: seedProjectId } },
+      prompt: "prompt {{variable}} {{anotherVariable}}",
+      isActive: true,
+      version: 1,
+      createdBy: "user-1",
+    },
+    update: {},
+  });
+
   const seedApiKey = {
     id: "seed-api-key",
     secret: process.env.SEED_SECRET_KEY ?? "sk-lf-1234567890",
@@ -79,9 +99,9 @@ async function main() {
 
   // Do not run the following for local docker compose setup
   if (environment === "examples") {
-    const project2 = await prisma.project.create({
-      data: {
-        id: "239ad00f-562f-411d-af14-831c75ddd875",
+    const project2 = await prisma.project.upsert({
+      where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+      create: {
         name: "demo-app",
         apiKeys: {
           create: [
@@ -100,16 +120,130 @@ async function main() {
           },
         },
       },
+      update: {},
     });
 
+    const promptIds: string[] = [];
+
+    const prompts = [
+      {
+        id: `prompt-${Math.floor(Math.random() * 1000000000)}`,
+        projectId: project2.id,
+        createdBy: "user-1",
+        prompt: "Prompt 1 content",
+        name: "Prompt 1",
+        version: 1,
+        isActive: true,
+      },
+      {
+        id: `prompt-${Math.floor(Math.random() * 1000000000)}`,
+        projectId: project2.id,
+        createdBy: "user-1",
+        prompt: "Prompt 2 content",
+        name: "Prompt 2",
+        version: 1,
+        isActive: true,
+      },
+      {
+        id: `prompt-${Math.floor(Math.random() * 1000000000)}`,
+        projectId: project2.id,
+        createdBy: "API",
+        prompt: "Prompt 3 content",
+        name: "Prompt 3 by API",
+        version: 1,
+        isActive: true,
+      },
+    ];
+
+    for (const prompt of prompts) {
+      await prisma.prompt.create({
+        data: {
+          id: prompt.id,
+          projectId: prompt.projectId,
+          createdBy: prompt.createdBy,
+          prompt: prompt.prompt,
+          name: prompt.name,
+          version: prompt.version,
+          isActive: prompt.isActive,
+        },
+      });
+      promptIds.push(prompt.id);
+    }
+
+    const promptVersionsWithVariables = [
+      {
+        id: `prompt-${Math.floor(Math.random() * 1000000000)}`,
+        projectId: project2.id,
+        createdBy: "user-1",
+        prompt: "Prompt 4 version 1 content with {{variable}}",
+        name: "Prompt 4 with variable",
+        version: 1,
+        isActive: false,
+      },
+      {
+        id: `prompt-${Math.floor(Math.random() * 1000000000)}`,
+        projectId: project2.id,
+        createdBy: "user-1",
+        prompt: "Prompt 4 version 2 content with {{variable}}",
+        name: "Prompt 4 with variable",
+        version: 2,
+        isActive: true,
+      },
+      {
+        id: `prompt-${Math.floor(Math.random() * 1000000000)}`,
+        projectId: project2.id,
+        createdBy: "user-1",
+        prompt: "Prompt 4 version 3 content with {{variable}}",
+        name: "Prompt 4 with variable",
+        version: 3,
+        isActive: false,
+      },
+    ];
+
+    for (const version of promptVersionsWithVariables) {
+      await prisma.prompt.create({
+        data: {
+          id: version.id,
+          projectId: version.projectId,
+          createdBy: version.createdBy,
+          prompt: version.prompt,
+          name: version.name,
+          version: version.version,
+          isActive: version.isActive,
+        },
+      });
+      promptIds.push(version.id);
+    }
+    const promptName = "Prompt with Longer Name";
+    const projectId = project2.id;
+    const createdBy = "user-1";
+
+    for (let i = 1; i <= 20; i++) {
+      const promptId = `prompt-${Math.floor(Math.random() * 1000000000)}`;
+      await prisma.prompt.create({
+        data: {
+          id: promptId,
+          projectId: projectId,
+          createdBy: createdBy,
+          prompt: `${promptName} version ${i} content`,
+          name: promptName,
+          version: i,
+          isActive: i === 20,
+        },
+      });
+      promptIds.push(promptId);
+    }
+
     const generationIds: string[] = [];
+    const envTags = [null, "development", "staging", "production"];
+    const colorTags = [null, "red", "blue", "yellow"];
 
     for (let i = 0; i < TRACE_VOLUME; i++) {
       // print progress to console with a progress bar that refreshes every 10 iterations
-      if (i % 10 === 0) {
+      if ((i + 1) % 10 === 0 || i === TRACE_VOLUME - 1) {
         process.stdout.clearLine(0);
         process.stdout.cursorTo(0);
-        process.stdout.write(`Seeding ${i} of ${TRACE_VOLUME}`);
+        process.stdout.write(`Seeding ${i + 1} of ${TRACE_VOLUME}`);
       }
       // random date within last 90 days, with a linear bias towards more recent dates
       const traceTs = new Date(
@@ -117,6 +251,13 @@ async function main() {
           Math.floor(Math.random() ** 1.5 * 90 * 24 * 60 * 60 * 1000),
       );
 
+      const envTag = envTags[Math.floor(Math.random() * envTags.length)];
+      const colorTag = colorTags[Math.floor(Math.random() * colorTags.length)];
+
+      const tags = [envTag, colorTag].filter((tag) => tag !== null);
+
+      const projectId = [project1.id, project2.id][i % 2] as string;
+
       const trace = await prisma.trace.create({
         data: {
           id: `trace-${Math.floor(Math.random() * 1000000000)}`,
@@ -127,25 +268,48 @@ async function main() {
           metadata: {
             user: `user-${i}@langfuse.com`,
           },
+          tags: tags as string[],
           project: {
-            connect: {
-              id: [project1.id, project2.id][i % 2],
-            },
+            connect: { id: projectId },
           },
           userId: `user-${i % 10}`,
+          session:
+            Math.random() > 0.3
+              ? {
+                  connectOrCreate: {
+                    where: {
+                      id_projectId: {
+                        id: `session-${i % 10}`,
+                        projectId: projectId,
+                      },
+                    },
+                    create: {
+                      id: `session-${i % 10}`,
+                      projectId: projectId,
+                    },
+                  },
+                }
+              : undefined,
+          input:
+            Math.random() > 0.3
+              ? "I'm looking for a React component"
+              : undefined,
+          output:
+            Math.random() > 0.3
+              ? "What kind of component are you looking for?"
+              : undefined,
           scores: {
             createMany: {
               data: [
-                {
-                  name: "latency",
-                  value: Math.floor(Math.random() * 20),
-                  timestamp: traceTs,
-                },
-                {
-                  name: "feedback",
-                  value: Math.floor(Math.random() * 3) - 1,
-                  timestamp: traceTs,
-                },
+                ...(Math.random() > 0.5
+                  ? [
+                      {
+                        name: "feedback",
+                        value: Math.floor(Math.random() * 3) - 1,
+                        timestamp: traceTs,
+                      },
+                    ]
+                  : []),
                 ...(Math.random() > 0.7
                   ? [
                       {
@@ -218,6 +382,20 @@ async function main() {
           const promptTokens = Math.floor(Math.random() * 1000) + 300;
           const completionTokens = Math.floor(Math.random() * 500) + 100;
 
+          const models = [
+            "gpt-3.5-turbo",
+            "gpt-4",
+            "gpt-4-32k-0613",
+            "gpt-3.5-turbo-16k-0613",
+            "claude-instant-1",
+            "claude-2.1",
+            "gpt-4-vision-preview",
+            "MIXTRAL-8X7B",
+          ];
+
+          const model = models[Math.floor(Math.random() * models.length)];
+          const promptId =
+            promptIds[Math.floor(Math.random() * promptIds.length)];
           const generation = await prisma.observation.create({
             data: {
               type: "GENERATION",
@@ -226,6 +404,7 @@ async function main() {
               endTime: generationTsEnd,
               name: `generation-${i}-${j}-${k}`,
               project: { connect: { id: trace.projectId } },
+              prompt: { connect: { id: promptId } },
               input:
                 Math.random() > 0.5
                   ? [
@@ -284,7 +463,8 @@ async function main() {
               
               Remember to import React at the top of your file whenever you're creating a component, because JSX transpiles to 'React.createElement' calls under the hood.`,
               },
-              model: Math.random() > 0.5 ? "gpt-3.5-turbo" : "gpt-4",
+              model: model,
+              internalModel: model,
               modelParameters: {
                 temperature:
                   Math.random() > 0.9 ? undefined : Math.random().toFixed(2),
@@ -303,25 +483,32 @@ async function main() {
               totalTokens: promptTokens + completionTokens,
               parentObservationId: span.id,
               traceId: trace.id,
+              ...{
+                ...(Math.random() > 0.5 && trace.projectId === prompt.projectId
+                  ? { prompt: { connect: { id: prompt.id } } }
+                  : {}),
+              },
+              unit: ModelUsageUnit.Tokens,
             },
           });
-
-          await prisma.score.create({
-            data: {
-              name: "quality",
-              value: Math.random() * 2 - 1,
-              observationId: generation.id,
-              traceId: trace.id,
-            },
-          });
-          await prisma.score.create({
-            data: {
-              name: "conciseness",
-              value: Math.random() * 2 - 1,
-              observationId: generation.id,
-              traceId: trace.id,
-            },
-          });
+          if (Math.random() > 0.6)
+            await prisma.score.create({
+              data: {
+                name: "quality",
+                value: Math.random() * 2 - 1,
+                observationId: generation.id,
+                traceId: trace.id,
+              },
+            });
+          if (Math.random() > 0.6)
+            await prisma.score.create({
+              data: {
+                name: "conciseness",
+                value: Math.random() * 2 - 1,
+                observationId: generation.id,
+                traceId: trace.id,
+              },
+            });
 
           generationIds.push(generation.id);
 
@@ -353,47 +540,54 @@ async function main() {
       }
     }
 
-    const dataset = await prisma.dataset.create({
-      data: {
-        name: "demo-dataset",
-        projectId: project2.id,
-      },
-    });
-
-    const datasetRun = await prisma.datasetRuns.create({
-      data: {
-        name: "demo-dataset-run",
-        datasetId: dataset.id,
-      },
-    });
-
-    for (let runNumber = 0; runNumber < 10; runNumber++) {
-      //pick randomly from existingSpanIds
-      const sourceObservationId =
-        generationIds[Math.floor(Math.random() * generationIds.length)];
-      const runObservationId =
-        generationIds[Math.floor(Math.random() * generationIds.length)];
-
-      const datasetItem = await prisma.datasetItem.create({
+    for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
+      const dataset = await prisma.dataset.create({
         data: {
-          datasetId: dataset.id,
-          sourceObservationId:
-            Math.random() > 0.5 ? sourceObservationId : undefined,
-          input: [
-            { role: "user", content: "How can i create a React component?" },
-          ],
-          expectedOutput:
-            "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.",
+          name: `demo-dataset-${datasetNumber}`,
+          projectId: project2.id,
         },
       });
 
-      await prisma.datasetRunItems.create({
-        data: {
-          datasetItemId: datasetItem.id,
-          observationId: runObservationId!,
-          datasetRunId: datasetRun.id,
-        },
-      });
+      for (let datasetRunNumber = 0; datasetRunNumber < 2; datasetRunNumber++) {
+        const datasetRun = await prisma.datasetRuns.create({
+          data: {
+            name: `demo-dataset-run-${datasetRunNumber}`,
+            datasetId: dataset.id,
+          },
+        });
+
+        for (let runNumber = 0; runNumber < 10; runNumber++) {
+          //pick randomly from existingSpanIds
+          const sourceObservationId =
+            generationIds[Math.floor(Math.random() * generationIds.length)];
+          const runObservationId =
+            generationIds[Math.floor(Math.random() * generationIds.length)];
+
+          const datasetItem = await prisma.datasetItem.create({
+            data: {
+              datasetId: dataset.id,
+              sourceObservationId:
+                Math.random() > 0.5 ? sourceObservationId : undefined,
+              input: [
+                {
+                  role: "user",
+                  content: "How can i create a React component?",
+                },
+              ],
+              expectedOutput:
+                "Creating a React component can be done in two ways: as a functional component or as a class component. Let's start with a basic example of both.",
+            },
+          });
+
+          await prisma.datasetRunItems.create({
+            data: {
+              datasetItemId: datasetItem.id,
+              observationId: runObservationId!,
+              datasetRunId: datasetRun.id,
+            },
+          });
+        }
+      }
     }
   }
 }

scripts/model-match.ts

@@ -0,0 +1,250 @@
+// Description: New model definitions in Langfuse are automatically applied to new observations.
+// You can optionally run this script to apply new model definitions to existing observations.
+// See docs: https://langfuse.com/docs/deployment/self-host#migrate-models
+// Execute: `npm run models:migrate`
+
+import "dotenv/config";
+
+import { findModel } from "@/src/server/api/services/EventProcessor";
+import { prisma } from "@/src/server/db";
+import lodash from "lodash";
+import { tokenCount } from "@/src/features/ingest/lib/usage";
+import { type Prisma } from "@prisma/client";
+
+async function main() {
+  return await modelMatch();
+}
+
+// Call the function
+main().catch((err) => {
+  console.error("An error occurred:", err);
+});
+
+export async function modelMatch() {
+  console.log("Starting model match");
+  const start = Date.now();
+
+  const BATCH_SIZE = 10_000;
+  let continueLoop = true;
+  let index = 0;
+  let totalObservations = 0;
+
+  while (continueLoop) {
+    type ObservationSelect = {
+      model: string | null;
+      id: string;
+      projectId: string;
+      startTime: Date;
+      unit: string | null;
+      promptTokens: number;
+      completionTokens: number;
+      totalTokens: number;
+      input: Prisma.JsonValue;
+      output: Prisma.JsonValue;
+    };
+
+    const observations = await prisma.observation.findMany({
+      select: {
+        id: true,
+        startTime: true,
+        model: true,
+        unit: true,
+        projectId: true,
+        promptTokens: true,
+        completionTokens: true,
+        totalTokens: true,
+        input: true,
+        output: true,
+      },
+      orderBy: {
+        startTime: "desc",
+      },
+      where: {
+        internalModel: null,
+        type: "GENERATION",
+      },
+      take: BATCH_SIZE,
+      skip: index * BATCH_SIZE,
+    });
+
+    console.log(`Found ${observations.length} observations to migrate`);
+
+    type Config = {
+      startTime: Date;
+      model: string;
+      unit: string;
+      projectId: string | null;
+    };
+
+    interface GroupedObservations {
+      [key: string]: ObservationSelect[];
+    }
+
+    const groupedObservations = observations.reduce<GroupedObservations>(
+      (acc, observation) => {
+        const config = {
+          startTime: observation.startTime.toISOString().slice(0, 10),
+          model: observation.model,
+          unit: observation.unit,
+          projectId: observation.projectId,
+        };
+
+        const key = JSON.stringify(config);
+
+        // Ensure the array is initialized before using it
+        acc[key] = acc[key] ?? [];
+        acc[key]?.push(observation);
+
+        return acc;
+      },
+      {},
+    );
+
+    let updatedObservations = 0;
+    const dbPromises = [];
+
+    for (const [key, observationsGroup] of Object.entries(
+      groupedObservations,
+    )) {
+      const { startTime, model, unit, projectId } = JSON.parse(key) as Config;
+
+      console.log("Execute key: ", startTime, model, unit, projectId);
+
+      if (!projectId) {
+        throw new Error("No project id");
+      }
+
+      const foundModel = await findModel({
+        event: { projectId, model, unit, startTime: startTime },
+      });
+
+      console.log(
+        "Found model: ",
+        foundModel?.id,
+        " for key: ",
+        key,
+        " with observations: ",
+        observationsGroup.length,
+      );
+
+      if (foundModel) {
+        // find all the observations with all tokens 0 and tokenize them individually
+        const observationsWithAllTokensZero = observationsGroup.filter(
+          (observation) =>
+            observation.promptTokens === 0 &&
+            observation.completionTokens === 0 &&
+            observation.totalTokens === 0,
+        );
+
+        for (const observation of observationsWithAllTokensZero) {
+          console.log("Tokenizing observation: ", observation.id);
+          const newInputCount = tokenCount({
+            model: foundModel,
+            text: observation.input,
+          });
+          const newOutputCount = tokenCount({
+            model: foundModel,
+            text: observation.output,
+          });
+
+          dbPromises.push(
+            prisma.observation.update({
+              where: {
+                id: observation.id,
+              },
+              data: {
+                promptTokens: newInputCount,
+                completionTokens: newOutputCount,
+                totalTokens: (newInputCount ?? 0) + (newOutputCount ?? 0),
+                internalModel: foundModel.modelName,
+              },
+            }),
+          );
+        }
+
+        // for all remaining observations, batch update them with the model id
+        const observationsWithTokens = observationsGroup.filter(
+          (observation) =>
+            observation.promptTokens !== 0 ||
+            observation.completionTokens !== 0 ||
+            observation.totalTokens !== 0,
+        );
+
+        // Push the promise for updating observations into the array
+        lodash.chunk(observationsWithTokens, 32000).map((chunk) => {
+          dbPromises.push(
+            prisma.observation.updateMany({
+              where: {
+                id: {
+                  in: chunk.map((observation) => observation.id),
+                },
+              },
+              data: {
+                internalModel: foundModel.modelName,
+              },
+            }),
+          );
+        });
+
+        updatedObservations += observationsGroup.length;
+      } else {
+        lodash.chunk(observationsGroup, 32000).map((chunk) => {
+          dbPromises.push(
+            prisma.observation.updateMany({
+              where: {
+                id: {
+                  in: chunk.map((observation) => observation.id),
+                },
+              },
+              data: {
+                internalModel: "LANGFUSETMPNOMODEL",
+              },
+            }),
+          );
+        });
+
+        updatedObservations += observationsGroup.length;
+      }
+    }
+
+    totalObservations += updatedObservations;
+    // Wait for all update operations to complete
+    const promiseChunk = lodash.chunk(dbPromises, 10);
+
+    for (const promises of promiseChunk) {
+      console.log("Waiting for promises to complete", promises.length);
+      await Promise.all(promises);
+    }
+
+    console.log(
+      "Updated observations count: ",
+      updatedObservations,
+      " in total: ",
+      totalObservations,
+    );
+
+    console.log(updatedObservations, observations.length);
+
+    if (updatedObservations === 0) {
+      index++;
+    }
+
+    if (observations.length === 0) {
+      console.log("No more observations to migrate");
+      continueLoop = false;
+    }
+  }
+
+  await prisma.observation.updateMany({
+    where: {
+      internalModel: "LANGFUSETMPNOMODEL",
+    },
+    data: {
+      internalModel: null,
+    },
+  });
+
+  const end = Date.now();
+
+  console.log(`Model match took ${end - start} ms`);
+}

security.md

@@ -1,11 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-We always recommend using the latest version of Langfuse to ensure you get all security updates.
-
-## Reporting a Vulnerability
-
-Please report security vulnerabilities to security@langfuse.com.
-
-We currently do not operate a bug bounty program.

sentry.client.config.ts

@@ -9,7 +9,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     // Set tracesSampleRate to 1.0 to capture 100%
     // of transactions for performance monitoring.
     // We recommend adjusting this value in production
-    tracesSampleRate: 0.8,
+    tracesSampleRate: 0.3,
 
     // Capture Replay for 100% of all sessions,
     // plus for 100% of sessions with an error

sentry.edge.config.ts

@@ -7,7 +7,7 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     // Set tracesSampleRate to 1.0 to capture 100%
     // of transactions for performance monitoring.
     // We recommend adjusting this value in production
-    tracesSampleRate: 0.75,
+    tracesSampleRate: 0.3,
 
     // ...
 

sentry.server.config.ts

@@ -12,9 +12,9 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
     // Set tracesSampleRate to 1.0 to capture 100%
     // of transactions for performance monitoring.
     // We recommend adjusting this value in production
-    tracesSampleRate: 0.8,
+    tracesSampleRate: 0.3,
 
-    profilesSampleRate: 0.8, // Profiling sample rate is relative to tracesSampleRate
+    profilesSampleRate: 0.3, // Profiling sample rate is relative to tracesSampleRate
     integrations: [
       // Add profiling integration to list of integrations
       new ProfilingIntegration(),

src/__e2e__/auth.spec.ts

@@ -2,6 +2,7 @@ import { test, expect } from "@playwright/test";
 
 test("should redirect to sign-in if not signed in", async ({ page }) => {
   await page.goto("/");
+  await page.waitForTimeout(2000);
   await expect(page).toHaveURL("/auth/sign-in");
 });
 

src/__e2e__/create-project.spec.ts

@@ -3,28 +3,33 @@ import { test, expect } from "@playwright/test";
 test("should see new projects dialog open after clicking new project btn", async ({
   page,
 }) => {
-  await page.goto("/auth/sign-in");
-  await page.fill('input[name="email"]', "demo@langfuse.com");
+  await page.goto("auth/sign-up");
+  await page.fill('input[name="name"]', "demo user");
+  await page.fill('input[name="email"]', randomEmailAddress());
   await page.fill('input[type="password"]', "password");
   await page.click('button[type="submit"]');
   await page.waitForTimeout(2000);
   await page.waitForTimeout(2000);
-  await page.isVisible("Create new project");
+  expect(await page.innerHTML("data-testid=create-new-project-title")).toBe(
+    "Create new project",
+  );
   await page.click('[data-testid="create-project-btn"]');
   await page.waitForTimeout(2000);
-  await page.isVisible('[data-testid="new-project-form"]');
+  await expect(page.locator("data-testid=new-project-form")).toBeVisible();
 });
 
 test("Create a project with provided name", async ({ page }) => {
+  test.setTimeout(60000);
+
   await page.goto("/auth/sign-in");
   await page.fill('input[name="email"]', "demo@langfuse.com");
   await page.fill('input[type="password"]', "password");
   await page.click('button[type="submit"]');
   await page.waitForTimeout(2000);
-  await page.isVisible("Create new project");
+  await page.isVisible('[data-testid="new-project-form"]');
   await page.click('[data-testid="create-project-btn"]');
   await page.waitForTimeout(2000);
-  await page.isVisible('[data-testid="new-project-form"]');
+  await expect(page.locator("data-testid=new-project-form")).toBeVisible();
   await page.fill(
     '[data-testid="new-project-name-input"]',
     "my e2e demo project",
@@ -33,7 +38,11 @@ test("Create a project with provided name", async ({ page }) => {
   await page.waitForTimeout(2000);
   expect(page.url()).toContain("/project/");
   await page.waitForTimeout(2000);
-  expect(await page.getByTestId("project-title-span-1").textContent()).toBe(
+  expect(await page.getByTestId("project-name").textContent()).toContain(
     "my e2e demo project",
   );
 });
+
+// random email address to be used in tests
+const randomEmailAddress = () =>
+  Math.random().toString(36).substring(2, 11) + "@example.com";

src/__tests__/cost.servertest.ts

@@ -0,0 +1,442 @@
+/** @jest-environment node */
+
+import { pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
+import { prisma } from "@/src/server/db";
+
+describe("cost retrieval tests", () => {
+  beforeEach(async () => await pruneDatabase());
+
+  [
+    {
+      testDescription: "prompt and completion tokens",
+      promptTokens: 200,
+      completionTokens: 3000,
+      totalTokens: undefined,
+      inputPrice: "0.0000010",
+      outputPrice: "0.0000020",
+      totalPrice: undefined,
+      expectedPromptTokens: 200,
+      expectedCompletionTokens: 3000,
+      expectedTotalTokens: 0,
+      expectedInputCost: "0.0002", // 200 / 1000 * 0.0010
+      expectedOutputCost: "0.006", // 3000 / 1000 * 0.0020
+      expectedTotalCost: "0.0062", // 0.0002 + 0.006
+    },
+    {
+      testDescription: "missing completion tokens",
+      promptTokens: 200,
+      completionTokens: undefined,
+      totalTokens: undefined,
+      inputPrice: "0.0000010",
+      outputPrice: "0.0000020",
+      totalPrice: undefined,
+      expectedPromptTokens: 200,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 0,
+      expectedInputCost: "0.0002",
+      expectedOutputCost: "0", // completionTokens are set to 0 when ingesting undefined, hence 0 cost
+      expectedTotalCost: "0.0002",
+    },
+    {
+      testDescription: "missing prompt tokens",
+      promptTokens: undefined,
+      completionTokens: 3000,
+      totalTokens: undefined,
+      inputPrice: "0.0000010",
+      outputPrice: "0.0000020",
+      totalPrice: undefined,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 3000,
+      expectedTotalTokens: 0,
+      expectedInputCost: "0", // promptTokens are set to 0 when ingesting undefined, hence 0 cost
+      expectedOutputCost: "0.006",
+      expectedTotalCost: "0.006",
+    },
+    {
+      testDescription: "prompt and completion and total",
+      promptTokens: 200,
+      completionTokens: 3000,
+      totalTokens: 3200,
+      inputPrice: "0.0000010",
+      outputPrice: "0.0000020",
+      totalPrice: undefined,
+      expectedPromptTokens: 200,
+      expectedCompletionTokens: 3000,
+      expectedTotalTokens: 3200,
+      expectedInputCost: "0.0002", // 200 / 1000 * 0.0010
+      expectedOutputCost: "0.006", // 3000 / 1000 * 0.0020
+      expectedTotalCost: "0.0062", // 0.0002 + 0.006
+    },
+    {
+      testDescription: "total only without price",
+      promptTokens: undefined,
+      completionTokens: undefined,
+      totalTokens: 3200,
+      inputPrice: "0.0000010",
+      outputPrice: "0.0000020",
+      totalPrice: undefined,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 3200,
+      expectedInputCost: "0",
+      expectedOutputCost: "0",
+      expectedTotalCost: "0",
+    },
+    {
+      testDescription: "total only",
+      promptTokens: undefined,
+      completionTokens: undefined,
+      totalTokens: 3200,
+      inputPrice: "0.0000010",
+      outputPrice: "0.0000020",
+      totalPrice: "0.1",
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 3200,
+      expectedInputCost: "0",
+      expectedOutputCost: "0",
+      expectedTotalCost: "320",
+    },
+  ].forEach((input) => {
+    it(`should calculate cost correctly ${input.testDescription}`, async () => {
+      await pruneDatabase();
+
+      await prisma.model.create({
+        data: {
+          modelName: "gpt-3.5-turbo",
+          inputPrice: input.inputPrice,
+          outputPrice: input.outputPrice,
+          totalPrice: input.totalPrice,
+          matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+          projectId: null,
+          startDate: new Date("2023-12-01"),
+          tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+          unit: ModelUsageUnit.Tokens,
+        },
+      });
+
+      const dbTrace = await prisma.trace.create({
+        data: {
+          name: "trace-name",
+          project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        },
+      });
+
+      await prisma.observation.create({
+        data: {
+          traceId: dbTrace.id,
+          type: "GENERATION",
+          project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+          model: "gpt-3.5-turbo",
+          internalModel: "gpt-3.5-turbo",
+          startTime: new Date("2024-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          promptTokens: input.promptTokens,
+          completionTokens: input.completionTokens,
+          totalTokens: input.totalTokens,
+        },
+      });
+
+      const view = await prisma.observationView.findFirst({
+        where: { traceId: dbTrace.id },
+      });
+
+      expect(view?.promptTokens).toBe(input.expectedPromptTokens);
+      expect(view?.completionTokens).toBe(input.expectedCompletionTokens);
+      expect(view?.totalTokens).toBe(input.expectedTotalTokens);
+
+      // calculated cost fields
+      expect(view?.calculatedInputCost?.toString()).toBe(
+        input.expectedInputCost,
+      );
+      expect(view?.calculatedOutputCost?.toString()).toBe(
+        input.expectedOutputCost,
+      );
+      expect(view?.calculatedTotalCost?.toString()).toBe(
+        input.expectedTotalCost,
+      );
+    });
+  });
+
+  [
+    {
+      testDescription: "overwriting project specific model",
+      expectedInputCost: "0.0004", // 200 / 1000 * 0.0010
+      expectedOutputCost: "0.012", // 3000 / 1000 * 0.0020
+      expectedTotalCost: "0.0124", // 0.0002 + 0.006
+      expectedModelId: "model-2",
+    },
+  ].forEach((input) => {
+    it(`should calculate cost correctly with multiple models ${input.testDescription}`, async () => {
+      await pruneDatabase();
+
+      await prisma.model.create({
+        data: {
+          id: "model-1",
+          modelName: "gpt-3.5-turbo",
+          inputPrice: "0.0000010",
+          outputPrice: "0.0000020",
+          totalPrice: "0.1",
+          matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+          projectId: null,
+          startDate: new Date("2023-12-01"),
+          tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+          unit: ModelUsageUnit.Tokens,
+        },
+      });
+      await prisma.model.create({
+        data: {
+          id: "model-2",
+          modelName: "gpt-3.5-turbo",
+          inputPrice: "0.0000020",
+          outputPrice: "0.0000040",
+          totalPrice: undefined,
+          matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+          startDate: new Date("2023-12-01"),
+          tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+          unit: ModelUsageUnit.Tokens,
+          project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        },
+      });
+
+      const dbTrace = await prisma.trace.create({
+        data: {
+          name: "trace-name",
+          project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        },
+      });
+
+      await prisma.observation.create({
+        data: {
+          traceId: dbTrace.id,
+          type: "GENERATION",
+          project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+          model: "gpt-3.5-turbo",
+          internalModel: "gpt-3.5-turbo",
+          startTime: new Date("2024-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          promptTokens: 200,
+          completionTokens: 3000,
+          totalTokens: undefined,
+        },
+      });
+
+      const view = await prisma.observationView.findFirst({
+        where: { traceId: dbTrace.id },
+      });
+
+      // calculated cost fields
+      expect(view?.modelId).toBe(input.expectedModelId);
+      expect(view?.calculatedInputCost?.toString()).toBe(
+        input.expectedInputCost,
+      );
+      expect(view?.calculatedOutputCost?.toString()).toBe(
+        input.expectedOutputCost,
+      );
+      expect(view?.calculatedTotalCost?.toString()).toBe(
+        input.expectedTotalCost,
+      );
+    });
+  });
+  it(`should prioritize latest models`, async () => {
+    await pruneDatabase();
+
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000010",
+        outputPrice: "0.0000020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        projectId: null,
+        startDate: new Date("2023-12-01"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000020",
+        outputPrice: "0.0000040",
+        totalPrice: undefined,
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-02"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+
+    const dbTrace = await prisma.trace.create({
+      data: {
+        name: "trace-name",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+      },
+    });
+
+    await prisma.observation.create({
+      data: {
+        traceId: dbTrace.id,
+        type: "GENERATION",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        model: "gpt-3.5-turbo",
+        internalModel: "gpt-3.5-turbo",
+        startTime: new Date("2024-01-01T00:00:00.000Z"),
+        unit: ModelUsageUnit.Tokens,
+        promptTokens: 200,
+        completionTokens: 3000,
+        totalTokens: undefined,
+      },
+    });
+
+    const view = await prisma.observationView.findFirst({
+      where: { traceId: dbTrace.id },
+    });
+
+    console.log(view);
+
+    // calculated cost fields
+    expect(view?.modelId).toBe("model-2");
+    expect(view?.calculatedInputCost?.toString()).toBe("0.0004");
+    expect(view?.calculatedOutputCost?.toString()).toBe("0.012");
+    expect(view?.calculatedTotalCost?.toString()).toBe("0.0124");
+  });
+
+  it(`should prioritize old model if the latest model is not own one`, async () => {
+    await pruneDatabase();
+
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0010",
+        outputPrice: "0.0020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-02"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000020",
+        outputPrice: "0.0000040",
+        totalPrice: undefined,
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-01"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+
+    const dbTrace = await prisma.trace.create({
+      data: {
+        name: "trace-name",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+      },
+    });
+
+    await prisma.observation.create({
+      data: {
+        traceId: dbTrace.id,
+        type: "GENERATION",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        model: "gpt-3.5-turbo",
+        internalModel: "gpt-3.5-turbo",
+        startTime: new Date("2024-01-01T00:00:00.000Z"),
+        unit: ModelUsageUnit.Tokens,
+        promptTokens: 200,
+        completionTokens: 3000,
+        totalTokens: undefined,
+      },
+    });
+
+    const view = await prisma.observationView.findFirst({
+      where: { traceId: dbTrace.id },
+    });
+
+    console.log(view);
+
+    // calculated cost fields
+    expect(view?.modelId).toBe("model-2");
+    expect(view?.calculatedInputCost?.toString()).toBe("0.0004");
+    expect(view?.calculatedOutputCost?.toString()).toBe("0.012");
+    expect(view?.calculatedTotalCost?.toString()).toBe("0.0124");
+  });
+
+  it(`should prioritize user provided cost`, async () => {
+    await pruneDatabase();
+
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0010",
+        outputPrice: "0.0020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-02"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0020",
+        outputPrice: "0.0040",
+        totalPrice: undefined,
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-01"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+
+    const dbTrace = await prisma.trace.create({
+      data: {
+        name: "trace-name",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+      },
+    });
+
+    await prisma.observation.create({
+      data: {
+        traceId: dbTrace.id,
+        type: "GENERATION",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        model: "gpt-3.5-turbo",
+        internalModel: "gpt-3.5-turbo",
+        startTime: new Date("2024-01-01T00:00:00.000Z"),
+        unit: ModelUsageUnit.Tokens,
+        promptTokens: 200,
+        completionTokens: 3000,
+        totalTokens: undefined,
+        inputCost: "1",
+        outputCost: "2",
+        totalCost: "3",
+      },
+    });
+
+    const view = await prisma.observationView.findFirst({
+      where: { traceId: dbTrace.id },
+    });
+
+    console.log(view);
+
+    // calculated cost fields
+    expect(view?.modelId).toBe("model-2");
+    expect(view?.calculatedInputCost?.toString()).toBe("1");
+    expect(view?.calculatedOutputCost?.toString()).toBe("2");
+    expect(view?.calculatedTotalCost?.toString()).toBe("3");
+  });
+});

src/__tests__/datasets.servertest.ts

@@ -0,0 +1,187 @@
+/** @jest-environment node */
+
+import { prisma } from "@/src/server/db";
+import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { v4 } from "uuid";
+
+describe("/api/public/datasets and /api/public/dataset-items API Endpoints", () => {
+  beforeEach(async () => await pruneDatabase());
+  afterEach(async () => await pruneDatabase());
+
+  it("should create and get a dataset", async () => {
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name",
+    });
+
+    const dbDataset = await prisma.dataset.findMany({
+      where: {
+        name: "dataset-name",
+      },
+    });
+
+    expect(dbDataset.length).toBeGreaterThan(0);
+
+    const getDataset = await makeAPICall(
+      "GET",
+      `/api/public/datasets/dataset-name`,
+    );
+
+    expect(getDataset.status).toBe(200);
+    expect(getDataset.body).toMatchObject({
+      name: "dataset-name",
+    });
+  });
+
+  it("should create and get a dataset item (via datasets and individually)", async () => {
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name",
+    });
+    await makeAPICall("POST", "/api/public/dataset-items", {
+      datasetName: "dataset-name",
+      input: { key: "value" },
+      expectedOutput: { key: "value" },
+    });
+    const dbDatasetItem = await prisma.datasetItem.findFirst({
+      where: {
+        dataset: {
+          name: "dataset-name",
+        },
+      },
+    });
+
+    expect(dbDatasetItem).not.toBeNull();
+
+    const getDataset = await makeAPICall(
+      "GET",
+      `/api/public/datasets/dataset-name`,
+    );
+    expect(getDataset.status).toBe(200);
+    expect(getDataset.body).toMatchObject({
+      name: "dataset-name",
+      items: [
+        {
+          id: dbDatasetItem!.id,
+          input: { key: "value" },
+          expectedOutput: { key: "value" },
+        },
+      ],
+    });
+
+    const getDatasetItem = await makeAPICall(
+      "GET",
+      `/api/public/dataset-items/${dbDatasetItem!.id}`,
+    );
+    expect(getDatasetItem.status).toBe(200);
+    expect(getDatasetItem.body).toMatchObject({
+      id: dbDatasetItem!.id,
+      input: { key: "value" },
+      expectedOutput: { key: "value" },
+    });
+  });
+
+  it("should upsert a dataset item", async () => {
+    await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name",
+    });
+
+    const item1 = await makeAPICall("POST", "/api/public/dataset-items", {
+      id: "dataset-item-id",
+      datasetName: "dataset-name",
+      input: { key: "value" },
+    });
+    expect(item1.status).toBe(200);
+    expect(item1.body).toMatchObject({
+      id: "dataset-item-id",
+    });
+
+    const item2 = await makeAPICall("POST", "/api/public/dataset-items", {
+      id: "dataset-item-id",
+      datasetName: "dataset-name",
+      input: { key: "value2" },
+    });
+    expect(item2.status).toBe(200);
+    expect(item2.body).toMatchObject({
+      id: "dataset-item-id",
+      input: { key: "value2" },
+    });
+
+    const dbDatasetItem = await prisma.datasetItem.findFirst({
+      where: { id: "dataset-item-id" },
+    });
+    expect(dbDatasetItem).not.toBeNull();
+    expect(dbDatasetItem?.input).toMatchObject({ key: "value2" });
+  });
+
+  it("should create and get a dataset run", async () => {
+    const dataset = await makeAPICall("POST", "/api/public/datasets", {
+      name: "dataset-name",
+    });
+    expect(dataset.status).toBe(200);
+    expect(dataset.body).toMatchObject({
+      name: "dataset-name",
+    });
+    await makeAPICall("POST", "/api/public/dataset-items", {
+      datasetName: "dataset-name",
+      id: "dataset-item-id",
+      input: { key: "value" },
+      expectedOutput: { key: "value" },
+    });
+    const traceId = v4();
+    const observationId = v4();
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name: "trace-name",
+            userId: "user-1",
+            metadata: { key: "value" },
+            release: "1.0.0",
+            version: "2.0.0",
+          },
+        },
+        {
+          id: v4(),
+          type: "observation-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: observationId,
+            traceId: traceId,
+            type: "GENERATION",
+            name: "generation-name",
+            startTime: "2021-01-01T00:00:00.000Z",
+            endTime: "2021-01-01T00:00:00.000Z",
+            modelParameters: { key: "value" },
+            input: { key: "value" },
+            metadata: { key: "value" },
+            version: "2.0.0",
+          },
+        },
+      ],
+    });
+    expect(response.status).toBe(207);
+
+    const runItem = await makeAPICall("POST", "/api/public/dataset-run-items", {
+      datasetItemId: "dataset-item-id",
+      observationId: observationId,
+      runName: "run-name",
+    });
+    const dbRun = await prisma.datasetRuns.findFirst({
+      where: {
+        name: "run-name",
+      },
+    });
+    expect(dbRun).not.toBeNull();
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    expect(dbRun?.datasetId).toBe(dataset.body.id);
+    expect(runItem.status).toBe(200);
+    expect(runItem.body).toMatchObject({
+      datasetItemId: "dataset-item-id",
+      observationId: observationId,
+      datasetRunId: dbRun?.id,
+    });
+  });
+});

src/__tests__/events.servertest.ts

@@ -241,12 +241,12 @@ describe("/api/public/events API Endpoint", () => {
 
     expect(createSpan.status).toBe(200);
 
-    expect(dbEvent?.id).toBe(spanId);
-    expect(dbEvent?.traceId).toBe(dbTrace[0]?.id);
-    expect(dbEvent?.name).toBe(generationName);
-    expect(dbEvent?.startTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
-    expect(dbEvent?.input).toEqual({ key: "value" });
-    expect(dbEvent?.metadata).toEqual({ key: "value" });
-    expect(dbEvent?.version).toBe("2.0.0");
+    expect(dbEvent.id).toBe(spanId);
+    expect(dbEvent.traceId).toBe(dbTrace[0]?.id);
+    expect(dbEvent.name).toBe(generationName);
+    expect(dbEvent.startTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
+    expect(dbEvent.input).toEqual({ key: "value" });
+    expect(dbEvent.metadata).toEqual({ key: "value" });
+    expect(dbEvent.version).toBe("2.0.0");
   });
 });

src/__tests__/features/orderBy/hooks/useOrderByState.clienttest.tsx

@@ -0,0 +1,110 @@
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+import { cleanup, render } from "@testing-library/react";
+
+import { TestRouter } from "@/src/__tests__/fixtures/TestRouter";
+import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
+import { LocationMock } from "@jedmao/location";
+import NextAdapterPages from "next-query-params/pages";
+import { useRouter } from "next/router";
+import { QueryParamProvider } from "use-query-params";
+
+afterEach(cleanup);
+
+jest.mock("next/router", () => ({
+  useRouter: jest.fn(),
+}));
+
+const { location: savedLocation } = window;
+
+// The test for the useOrderByState hook
+describe("useOrderByState hook", () => {
+  let testRouter: TestRouter;
+  let locationMock: LocationMock;
+
+  beforeAll(() => {
+    // @ts-expect-error - TS only allows to delete optional params; but here we want to delete it
+    // as it will be set in beforeEach
+    delete window.location;
+  });
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+    locationMock = new LocationMock("https://langfuse.com");
+    testRouter = new TestRouter(locationMock);
+    window.location = locationMock;
+
+    (useRouter as jest.Mock).mockReturnValue(testRouter);
+  });
+
+  afterAll(() => {
+    window.location = savedLocation;
+  });
+
+  test("orderBy takes the default value if no url param is given", () => {
+    const result = render(
+      <QueryParamProvider adapter={NextAdapterPages}>
+        <UseOrderByStateExample />
+      </QueryParamProvider>,
+    );
+
+    expect(result.queryByText(/column Column 1, order DESC/)).toBeTruthy();
+  });
+
+  test("setOrderBy updates orderBy query param", () => {
+    const result = render(
+      <QueryParamProvider adapter={NextAdapterPages}>
+        <UseOrderByStateExample />
+      </QueryParamProvider>,
+    );
+
+    result.queryByText(/Set Order/)!.click();
+    result.rerender(
+      <QueryParamProvider adapter={NextAdapterPages}>
+        <UseOrderByStateExample />
+      </QueryParamProvider>,
+    );
+
+    expect(result.queryByText(/column Column 2, order ASC/)).toBeTruthy();
+    expect(locationMock.searchParams.get("orderBy")).toBe(
+      "column-Column 2_order-ASC",
+    );
+  });
+
+  test("orderBy reads the given param from the url", () => {
+    locationMock.replace(
+      "https://langfuse.com?orderBy=column-Column 3_order-DESC",
+    );
+
+    const result = render(
+      <QueryParamProvider adapter={NextAdapterPages}>
+        <UseOrderByStateExample />
+      </QueryParamProvider>,
+    );
+
+    expect(result.queryByText(/column Column 3, order DESC/)).toBeTruthy();
+  });
+});
+
+const UseOrderByStateExample = () => {
+  const [orderBy, setOrderBy] = useOrderByState({
+    column: "Column 1",
+    order: "DESC",
+  });
+
+  return (
+    <>
+      <h1>{`column ${orderBy?.column}, order ${orderBy?.order}`}</h1>
+      <button
+        onClick={() => {
+          setOrderBy({
+            column: "Column 2",
+            order: "ASC",
+          });
+        }}
+      >
+        Set Order
+      </button>
+    </>
+  );
+};

src/__tests__/features/orderBy/server/orderByToPrisma.clienttest.ts

@@ -0,0 +1,29 @@
+import { orderByToPrismaSql } from "@/src/features/orderBy/server/orderByToPrisma";
+import { tracesTableCols } from "@/src/server/api/definitions/tracesTable";
+
+// The test for the orderByToPrisma function
+describe("orderByToPrisma (Convert orderBy to Prisma.sql)", () => {
+  test("orderByToPrisma throws error for orderBy column not included in column defs", () => {
+    expect(() =>
+      orderByToPrismaSql(
+        {
+          column: "InvalidCol",
+          order: "ASC",
+        },
+        tracesTableCols,
+      ),
+    ).toThrow(/Invalid filter column: InvalidCol/);
+  });
+
+  test("orderByToPrisma throws error for orderBy order that is not valid", () => {
+    expect(() =>
+      orderByToPrismaSql(
+        {
+          column: "latency",
+          order: "test" as "ASC" | "DESC",
+        },
+        tracesTableCols,
+      ),
+    ).toThrow(/Invalid order: test/);
+  });
+});

src/__tests__/fixtures/TestRouter.ts

@@ -0,0 +1,56 @@
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+import { type Router } from "next/router";
+import type { UrlObject } from "url";
+import { type LocationMock } from "@jedmao/location";
+
+type PartialRouter = Partial<Router>;
+
+export const BASE_URL = "https://langfuse.com";
+
+/**
+ * A Router to be used for testing which provides the bare minimum needed
+ * for the useQueryParam(s) hook and NextAdapter to work.
+ */
+export class TestRouter implements PartialRouter {
+  isReady = true;
+  pathname = "/";
+  private currentUrl = "";
+  private history: string[] = [];
+
+  constructor(private locationMock: LocationMock) {}
+
+  replace = (url: string | UrlObject) => {
+    // eslint-disable-next-line @typescript-eslint/no-base-to-string, @typescript-eslint/restrict-template-expressions
+    this.locationMock.assign(`${BASE_URL}${url}`);
+    this.currentUrl = TestRouter.getURLString(url);
+    this.locationMock.assign(`${BASE_URL}${this.currentUrl}`);
+    return Promise.resolve(true);
+  };
+
+  push = (url: string | UrlObject) => {
+    this.history.push(this.currentUrl);
+    this.currentUrl = TestRouter.getURLString(url);
+    this.locationMock.assign(`${BASE_URL}${this.currentUrl}`);
+    return Promise.resolve(true);
+  };
+
+  setIsReady = (isReady: boolean) => {
+    this.isReady = isReady;
+  };
+
+  get asPath() {
+    return this.pathname;
+  }
+
+  static getURLString(url: string | UrlObject): string {
+    if (typeof url === "string") {
+      return url;
+    }
+    return `${url.pathname}${url.search}`;
+  }
+
+  getParams(): URLSearchParams {
+    return new URL(`${BASE_URL}${this.currentUrl}`).searchParams;
+  }
+}

src/__tests__/generations.servertest.ts

@@ -1,74 +1,161 @@
 /** @jest-environment node */
 
-import { prisma } from "@/src/server/db";
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
 
+import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
+import { prisma } from "@/src/server/db";
+
 describe("/api/public/generations API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
-  afterEach(async () => await pruneDatabase());
 
-  it("should create generation after trace", async () => {
-    await pruneDatabase();
+  [
+    {
+      usage: {
+        input: 100,
+        output: 200,
+        total: 100,
+        unit: ModelUsageUnit.Characters,
+      },
+      expectedUnit: ModelUsageUnit.Characters,
+      expectedPromptTokens: 100,
+      expectedCompletionTokens: 200,
+      expectedTotalTokens: 100,
+    },
+    {
+      usage: {
+        total: 100,
+        unit: ModelUsageUnit.Characters,
+      },
+      expectedUnit: ModelUsageUnit.Characters,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 100,
+    },
+    {
+      usage: {
+        total: 100,
+      },
+      expectedUnit: null,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 100,
+    },
+    {
+      usage: {
+        promptTokens: 100,
+        completionTokens: 200,
+        totalTokens: 100,
+      },
+      expectedPromptTokens: 100,
+      expectedCompletionTokens: 200,
+      expectedTotalTokens: 100,
+      expectedUnit: ModelUsageUnit.Tokens,
+    },
+    {
+      usage: {
+        totalTokens: 100,
+      },
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 100,
+      expectedUnit: ModelUsageUnit.Tokens,
+    },
+    {
+      usage: undefined,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 0,
+      expectedUnit: null,
+    },
+    {
+      usage: null,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 0,
+      expectedUnit: null,
+    },
+    {
+      usage: {},
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 0,
+      expectedUnit: null,
+    },
+  ].forEach((testConfig) => {
+    it(`should create generation after trace 1 ${JSON.stringify(
+      testConfig,
+    )}`, async () => {
+      await pruneDatabase();
 
-    const traceId = uuidv4();
+      const traceId = uuidv4();
 
-    await makeAPICall("POST", "/api/public/traces", {
-      id: traceId,
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      metadata: { key: "value" },
-      release: "1.0.0",
-      version: "2.0.0",
-    });
-
-    const dbTrace = await prisma.trace.findMany({
-      where: {
+      await makeAPICall("POST", "/api/public/traces", {
         id: traceId,
-      },
-    });
-
-    expect(dbTrace.length).toBeGreaterThan(0);
-    expect(dbTrace[0]?.id).toBe(traceId);
-
-    const generationId = uuidv4();
-    const createGeneration = await makeAPICall(
-      "POST",
-      "/api/public/generations",
-      {
-        id: generationId,
-        traceId: traceId,
-        name: "generation-name",
-        startTime: "2021-01-01T00:00:00.000Z",
-        endTime: "2021-01-01T00:00:00.000Z",
-        model: "model-name",
-        modelParameters: { key: "value" },
-        prompt: { key: "value" },
+        name: "trace-name",
+        userId: "user-1",
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
         metadata: { key: "value" },
+        release: "1.0.0",
         version: "2.0.0",
-      },
-    );
+      });
 
-    expect(createGeneration.status).toBe(200);
-    const dbGeneration = await prisma.observation.findUnique({
-      where: {
-        id: generationId,
-      },
+      const dbTrace = await prisma.trace.findMany({
+        where: {
+          id: traceId,
+        },
+      });
+
+      expect(dbTrace.length).toBeGreaterThan(0);
+      expect(dbTrace[0]?.id).toBe(traceId);
+
+      const generationId = uuidv4();
+      const createGeneration = await makeAPICall(
+        "POST",
+        "/api/public/generations",
+        {
+          id: generationId,
+          traceId: traceId,
+          name: "generation-name",
+          startTime: "2021-01-01T00:00:00.000Z",
+          endTime: "2021-01-01T00:00:00.000Z",
+          model: "model-name",
+          modelParameters: { key: "value" },
+          prompt: { key: "value" },
+          metadata: { key: "value" },
+          version: "2.0.0",
+          usage: testConfig.usage,
+        },
+      );
+
+      expect(createGeneration.status).toBe(200);
+      const dbGeneration = await prisma.observation.findUnique({
+        where: {
+          id: generationId,
+        },
+      });
+
+      expect(dbGeneration?.id).toBe(generationId);
+      expect(dbGeneration?.traceId).toBe(traceId);
+      expect(dbGeneration?.name).toBe("generation-name");
+      expect(dbGeneration?.startTime).toEqual(
+        new Date("2021-01-01T00:00:00.000Z"),
+      );
+      expect(dbGeneration?.endTime).toEqual(
+        new Date("2021-01-01T00:00:00.000Z"),
+      );
+      expect(dbGeneration?.model).toBe("model-name");
+      expect(dbGeneration?.modelParameters).toEqual({ key: "value" });
+      expect(dbGeneration?.input).toEqual({ key: "value" });
+      expect(dbGeneration?.metadata).toEqual({ key: "value" });
+      expect(dbGeneration?.version).toBe("2.0.0");
+      expect(dbGeneration?.unit).toBe(testConfig.expectedUnit);
+      expect(dbGeneration?.promptTokens).toBe(testConfig.expectedPromptTokens);
+      expect(dbGeneration?.completionTokens).toBe(
+        testConfig.expectedCompletionTokens,
+      );
+      expect(dbGeneration?.totalTokens).toBe(testConfig.expectedTotalTokens);
     });
-
-    expect(dbGeneration?.id).toBe(generationId);
-    expect(dbGeneration?.traceId).toBe(traceId);
-    expect(dbGeneration?.name).toBe("generation-name");
-    expect(dbGeneration?.startTime).toEqual(
-      new Date("2021-01-01T00:00:00.000Z"),
-    );
-    expect(dbGeneration?.endTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
-    expect(dbGeneration?.model).toBe("model-name");
-    expect(dbGeneration?.modelParameters).toEqual({ key: "value" });
-    expect(dbGeneration?.input).toEqual({ key: "value" });
-    expect(dbGeneration?.metadata).toEqual({ key: "value" });
-    expect(dbGeneration?.version).toBe("2.0.0");
   });
 
   it("should create generation before trace", async () => {
@@ -339,8 +426,8 @@ describe("/api/public/generations API Endpoint", () => {
         name: generationName,
       },
     });
-    expect(dbGeneration?.id).toBe(generationId);
-    expect(dbGeneration?.traceId).toBe(externalTraceId);
+    expect(dbGeneration.id).toBe(generationId);
+    expect(dbGeneration.traceId).toBe(externalTraceId);
 
     const dbTraces = await prisma.trace.findMany();
     expect(dbTraces.length).toBe(0);
@@ -383,18 +470,18 @@ describe("/api/public/generations API Endpoint", () => {
 
     expect(createGeneration.status).toBe(200);
 
-    expect(dbGeneration?.id).toBe(generationId);
-    expect(dbGeneration?.traceId).toBe(dbTrace[0]?.id);
-    expect(dbGeneration?.name).toBe(generationName);
-    expect(dbGeneration?.startTime).toEqual(
+    expect(dbGeneration.id).toBe(generationId);
+    expect(dbGeneration.traceId).toBe(dbTrace[0]?.id);
+    expect(dbGeneration.name).toBe(generationName);
+    expect(dbGeneration.startTime).toEqual(
       new Date("2021-01-01T00:00:00.000Z"),
     );
-    expect(dbGeneration?.endTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
-    expect(dbGeneration?.model).toBe("model-name");
-    expect(dbGeneration?.modelParameters).toEqual({ key: "value" });
-    expect(dbGeneration?.input).toEqual({ key: "value" });
-    expect(dbGeneration?.metadata).toEqual({ key: "value" });
-    expect(dbGeneration?.version).toBe("2.0.0");
+    expect(dbGeneration.endTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
+    expect(dbGeneration.model).toBe("model-name");
+    expect(dbGeneration.modelParameters).toEqual({ key: "value" });
+    expect(dbGeneration.input).toEqual({ key: "value" });
+    expect(dbGeneration.metadata).toEqual({ key: "value" });
+    expect(dbGeneration.version).toBe("2.0.0");
   });
 
   it("should update generation", async () => {

src/__tests__/ingestion.servertest.ts

@@ -1,18 +1,565 @@
 /** @jest-environment node */
 
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
-import { prisma } from "@/src/server/db";
 import { v4 } from "uuid";
 
+import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
+import { cleanEvent } from "@/src/pages/api/public/ingestion";
+import { prisma } from "@/src/server/db";
+
 describe("/api/public/ingestion API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
 
-  it("should create trace and generation", async () => {
+  [
+    {
+      usage: {
+        input: 100,
+        output: 200,
+        total: 100,
+        unit: ModelUsageUnit.Characters,
+        inputCost: 123,
+        outputCost: 456,
+        totalCost: 789,
+      },
+      expectedUnit: ModelUsageUnit.Characters,
+      expectedPromptTokens: 100,
+      expectedCompletionTokens: 200,
+      expectedTotalTokens: 100,
+    },
+    {
+      usage: {
+        total: 100,
+        unit: ModelUsageUnit.Characters,
+      },
+      expectedUnit: ModelUsageUnit.Characters,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 100,
+    },
+    {
+      usage: {
+        total: 100,
+        unit: ModelUsageUnit.Milliseconds,
+      },
+      expectedUnit: ModelUsageUnit.Milliseconds,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 100,
+    },
+    {
+      usage: {
+        input: 1,
+        output: 2,
+        unit: ModelUsageUnit.Images,
+      },
+      expectedUnit: ModelUsageUnit.Images,
+      expectedPromptTokens: 1,
+      expectedCompletionTokens: 2,
+      expectedTotalTokens: 3,
+    },
+    {
+      usage: {
+        input: 30,
+        output: 10,
+        unit: ModelUsageUnit.Seconds,
+      },
+      expectedUnit: ModelUsageUnit.Seconds,
+      expectedPromptTokens: 30,
+      expectedCompletionTokens: 10,
+      expectedTotalTokens: 40,
+    },
+    {
+      usage: {
+        total: 100,
+      },
+      expectedUnit: null,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 100,
+    },
+    {
+      usage: {
+        promptTokens: 100,
+        completionTokens: 200,
+        totalTokens: 100,
+      },
+      expectedPromptTokens: 100,
+      expectedCompletionTokens: 200,
+      expectedTotalTokens: 100,
+      expectedUnit: ModelUsageUnit.Tokens,
+    },
+    {
+      usage: {
+        totalTokens: 100,
+      },
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 100,
+      expectedUnit: ModelUsageUnit.Tokens,
+    },
+    {
+      usage: undefined,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 0,
+      expectedUnit: null,
+    },
+    {
+      usage: null,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 0,
+      expectedUnit: null,
+    },
+    {
+      usage: {},
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 0,
+      expectedUnit: null,
+    },
+  ].forEach((testConfig) => {
+    it(`should create trace, generation and score without matching models ${JSON.stringify(
+      testConfig,
+    )}`, async () => {
+      const traceId = v4();
+      const generationId = v4();
+      const spanId = v4();
+      const scoreId = v4();
+
+      const response = await makeAPICall("POST", "/api/public/ingestion", {
+        metadata: {
+          sdk_verion: "1.0.0",
+          sdk_name: "python",
+        },
+        batch: [
+          {
+            id: v4(),
+            type: "trace-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: traceId,
+              name: "trace-name",
+              userId: "user-1",
+              metadata: { key: "value" },
+              release: "1.0.0",
+              version: "2.0.0",
+              tags: ["tag-1", "tag-2"],
+            },
+          },
+          {
+            id: v4(),
+            type: "observation-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: generationId,
+              traceId: traceId,
+              type: "GENERATION",
+              name: "generation-name",
+              startTime: "2021-01-01T00:00:00.000Z",
+              endTime: "2021-01-01T00:00:00.000Z",
+              modelParameters: { key: "value" },
+              input: { key: "value" },
+              metadata: { key: "value" },
+              version: "2.0.0",
+            },
+          },
+          {
+            id: v4(),
+            type: "observation-update",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: generationId,
+              type: "GENERATION",
+              output: { key: "this is a great gpt output" },
+              usage: testConfig.usage,
+            },
+          },
+          {
+            id: v4(),
+            type: "observation-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: spanId,
+              traceId: traceId,
+              type: "SPAN",
+              name: "span-name",
+              startTime: "2021-01-01T00:00:00.000Z",
+              endTime: "2021-01-01T00:00:00.000Z",
+              input: { input: "value" },
+              metadata: { meta: "value" },
+              version: "2.0.0",
+            },
+          },
+          {
+            id: v4(),
+            type: "score-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: scoreId,
+              name: "score-name",
+              value: 100.5,
+              traceId: traceId,
+            },
+          },
+        ],
+      });
+
+      expect(response.status).toBe(207);
+
+      console.log("response body", response.body);
+
+      const dbTrace = await prisma.trace.findMany({
+        where: {
+          name: "trace-name",
+        },
+      });
+
+      expect(dbTrace.length).toBeGreaterThan(0);
+      expect(dbTrace[0]?.name).toBe("trace-name");
+      expect(dbTrace[0]?.release).toBe("1.0.0");
+      expect(dbTrace[0]?.externalId).toBeNull();
+      expect(dbTrace[0]?.version).toBe("2.0.0");
+      expect(dbTrace[0]?.projectId).toBe(
+        "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      );
+      expect(dbTrace[0]?.tags).toEqual(["tag-1", "tag-2"]);
+
+      const dbGeneration = await prisma.observation.findUnique({
+        where: {
+          id: generationId,
+        },
+      });
+
+      expect(dbGeneration?.id).toBe(generationId);
+      expect(dbGeneration?.traceId).toBe(traceId);
+      expect(dbGeneration?.name).toBe("generation-name");
+      expect(dbGeneration?.startTime).toEqual(
+        new Date("2021-01-01T00:00:00.000Z"),
+      );
+      expect(dbGeneration?.endTime).toEqual(
+        new Date("2021-01-01T00:00:00.000Z"),
+      );
+      expect(dbGeneration?.model).toBeNull();
+      expect(dbGeneration?.modelParameters).toEqual({ key: "value" });
+      expect(dbGeneration?.input).toEqual({ key: "value" });
+      expect(dbGeneration?.metadata).toEqual({ key: "value" });
+      expect(dbGeneration?.version).toBe("2.0.0");
+      expect(dbGeneration?.internalModel).toBeNull();
+      expect(dbGeneration?.promptTokens).toEqual(
+        testConfig.expectedPromptTokens,
+      );
+      expect(dbGeneration?.completionTokens).toEqual(
+        testConfig.expectedCompletionTokens,
+      );
+      expect(dbGeneration?.totalTokens).toEqual(testConfig.expectedTotalTokens);
+      expect(dbGeneration?.unit).toEqual(testConfig.expectedUnit);
+      expect(dbGeneration?.output).toEqual({
+        key: "this is a great gpt output",
+      });
+
+      const dbSpan = await prisma.observation.findUnique({
+        where: {
+          id: spanId,
+        },
+      });
+
+      expect(dbSpan?.id).toBe(spanId);
+      expect(dbSpan?.name).toBe("span-name");
+      expect(dbSpan?.startTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
+      expect(dbSpan?.endTime).toEqual(new Date("2021-01-:00:00.000Z"));
+      expect(dbSpan?.input).toEqual({ input: "value" });
+      expect(dbSpan?.metadata).toEqual({ meta: "value" });
+      expect(dbSpan?.version).toBe("2.0.0");
+
+      const dbScore = await prisma.score.findUnique({
+        where: {
+          id: scoreId,
+        },
+      });
+
+      expect(dbScore?.id).toBe(scoreId);
+      expect(dbScore?.traceId).toBe(traceId);
+      expect(dbScore?.name).toBe("score-name");
+      expect(dbScore?.value).toBe(100.5);
+      expect(dbScore?.observationId).toBeNull();
+    });
+  });
+
+  [
+    {
+      observationExternalModel: "gpt-3.5",
+      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
+      modelUnit: ModelUsageUnit.Tokens,
+      expectedInternalModel: "gpt-3.5-turbo",
+      expectedPromptTokens: 5,
+      expectedCompletionTokens: 7,
+      models: [
+        {
+          modelName: "gpt-3.5-turbo",
+          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+          startDate: new Date("2021-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+      ],
+    },
+    {
+      observationExternalModel: "gpt-3.5",
+      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
+      modelUnit: ModelUsageUnit.Tokens,
+      expectedInternalModel: "gpt-3.5-turbo",
+      expectedPromptTokens: 5,
+      expectedCompletionTokens: 7,
+      models: [
+        {
+          modelName: "gpt-3.5-turbo",
+          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+          startDate: null,
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+      ],
+    },
+    {
+      observationExternalModel: "GPT-3.5",
+      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
+      modelUnit: ModelUsageUnit.Tokens,
+      expectedInternalModel: "gpt-3.5-turbo",
+      expectedPromptTokens: 5,
+      expectedCompletionTokens: 7,
+      models: [
+        {
+          modelName: "gpt-3.5-turbo",
+          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+          startDate: new Date("2021-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+      ],
+    },
+    {
+      observationExternalModel: "GPT-3.5",
+      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
+      modelUnit: ModelUsageUnit.Tokens,
+      expectedInternalModel: "gpt-3.5-turbo",
+      expectedPromptTokens: 5,
+      expectedCompletionTokens: 7,
+      models: [
+        {
+          modelName: "gpt-3.5-turbo",
+          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+          startDate: new Date("2021-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+        {
+          modelName: "gpt-3.5-turbo-new",
+          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+          startDate: new Date("2021-01-01T10:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+      ],
+    },
+    {
+      observationExternalModel: "GPT-3.5",
+      observationStartTime: new Date("2021-01-02T00:00:00.000Z"),
+      modelUnit: ModelUsageUnit.Tokens,
+      expectedInternalModel: "gpt-3.5-turbo",
+      expectedPromptTokens: 5,
+      expectedCompletionTokens: 7,
+      models: [
+        {
+          modelName: "gpt-3.5-turbo-new",
+          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+          startDate: new Date("2021-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+        {
+          modelName: "gpt-3.5-turbo",
+          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+          startDate: new Date("2021-01-01T10:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+          tokenizerModel: "gpt-3.5-turbo",
+        },
+      ],
+    },
+    {
+      observationExternalModel: "ft:gpt-3.5-turbo-1106:my-org:custom_suffix:id",
+      observationStartTime: new Date("2022-01-01T10:00:00.000Z"),
+      modelUnit: ModelUsageUnit.Tokens,
+      expectedInternalModel: "ft:gpt-3.5-turbo-1106",
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      models: [
+        {
+          modelName: "ft:gpt-3.5-turbo-1106",
+          matchPattern: "(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$",
+          startDate: new Date("2022-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+      ],
+    },
+    {
+      observationExternalModel: "ft:babbage-002:my-org#2:custom_suffix-2:id",
+      observationStartTime: new Date("2022-01-01T10:00:00.000Z"),
+      modelUnit: ModelUsageUnit.Tokens,
+      expectedInternalModel: "ft:babbage-002",
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      models: [
+        {
+          modelName: "ft:babbage-002",
+          matchPattern: "(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$",
+          startDate: new Date("2022-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+      ],
+    },
+    {
+      observationExternalModel: "GPT-4",
+      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
+      modelUnit: ModelUsageUnit.Tokens,
+      expectedInternalModel: null,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      models: [
+        {
+          modelName: "gpt-3.5-turbo",
+          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+          startDate: new Date("2021-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+      ],
+    },
+    {
+      observationExternalModel: "GPT-3",
+      observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
+      modelUnit: ModelUsageUnit.Characters,
+      expectedInternalModel: null,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      models: [
+        {
+          modelName: "gpt-3.5-turbo",
+          matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+          startDate: new Date("2021-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          tokenizerId: "openai",
+        },
+      ],
+    },
+  ].forEach((testConfig) => {
+    it(`should match observations to internal models ${JSON.stringify(
+      testConfig,
+    )}`, async () => {
+      const traceId = v4();
+      const generationId = v4();
+
+      await Promise.all(
+        testConfig.models.map(async (model) =>
+          prisma.model.create({
+            data: {
+              modelName: model.modelName,
+              matchPattern: model.matchPattern,
+              startDate: model.startDate,
+              unit: model.unit,
+              tokenizerId: model.tokenizerId,
+              tokenizerConfig: {
+                tokensPerMessage: 3,
+                tokensPerName: 1,
+                tokenizerModel:
+                  "tokenizerModel" in model
+                    ? model.tokenizerModel
+                    : model.modelName,
+              },
+            },
+          }),
+        ),
+      );
+
+      const response = await makeAPICall("POST", "/api/public/ingestion", {
+        metadata: {
+          sdk_verion: "1.0.0",
+          sdk_name: "python",
+        },
+        batch: [
+          {
+            id: v4(),
+            type: "trace-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: traceId,
+              name: "trace-name",
+            },
+          },
+          {
+            id: v4(),
+            type: "observation-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: generationId,
+              traceId: traceId,
+              type: "GENERATION",
+              name: "generation-name",
+              startTime: testConfig.observationStartTime.toISOString(),
+              model: testConfig.observationExternalModel,
+              usage: {
+                unit: testConfig.modelUnit,
+              },
+              input: "This is a great prompt",
+              output: "This is a great gpt output",
+            },
+          },
+        ],
+      });
+
+      expect(response.status).toBe(207);
+
+      console.log("response body", response.body);
+
+      const dbGeneration = await prisma.observation.findUnique({
+        where: {
+          id: generationId,
+        },
+      });
+
+      expect(dbGeneration?.id).toBe(generationId);
+      expect(dbGeneration?.traceId).toBe(traceId);
+      expect(dbGeneration?.name).toBe("generation-name");
+      expect(dbGeneration?.startTime).toEqual(testConfig.observationStartTime);
+      expect(dbGeneration?.model).toBe(testConfig.observationExternalModel);
+      expect(dbGeneration?.promptTokens).toBe(testConfig.expectedPromptTokens);
+      expect(dbGeneration?.completionTokens).toBe(
+        testConfig.expectedCompletionTokens,
+      );
+      expect(dbGeneration?.internalModel).toBe(
+        testConfig.expectedInternalModel,
+      );
+    });
+  });
+
+  it("should create and update all events", async () => {
     const traceId = v4();
     const generationId = v4();
     const spanId = v4();
+    const eventId = v4();
     const scoreId = v4();
 
+    const exception = `
+    ERROR    langfuse:callback.py:677 'model_name'
+    Traceback (most recent call last):
+      File "/Users/maximiliandeichmann/development/github.com/langfuse/langfuse-python/langfuse/callback.py", line 674, in __on_llm_action
+        model_name = kwargs["invocation_params"]["model_name"]
+    KeyError: 'model_name'
+    `;
+
     const response = await makeAPICall("POST", "/api/public/ingestion", {
       batch: [
         {
@@ -21,55 +568,56 @@ describe("/api/public/ingestion API Endpoint", () => {
           timestamp: new Date().toISOString(),
           body: {
             id: traceId,
-            name: "trace-name",
-            userId: "user-1",
-            metadata: { key: "value" },
-            release: "1.0.0",
-            version: "2.0.0",
           },
         },
         {
           id: v4(),
-          type: "observation-create",
+          type: "span-create",
           timestamp: new Date().toISOString(),
           body: {
-            id: generationId,
+            id: spanId,
             traceId: traceId,
-            type: "GENERATION",
-            name: "generation-name",
-            startTime: "2021-01-01T00:00:00.000Z",
-            endTime: "2021-01-01T00:00:00.000Z",
-            modelParameters: { key: "value" },
-            input: { key: "value" },
-            metadata: { key: "value" },
-            version: "2.0.0",
           },
         },
         {
           id: v4(),
-          type: "observation-update",
-          timestamp: new Date().toISOString(),
-          body: {
-            id: generationId,
-            type: "GENERATION",
-            output: { key: "this is a great gpt output" },
-            usage: { promptTokens: 400, completionTokens: 1000 },
-          },
-        },
-        {
-          id: v4(),
-          type: "observation-create",
+          type: "span-update",
           timestamp: new Date().toISOString(),
           body: {
             id: spanId,
             traceId: traceId,
-            type: "SPAN",
             name: "span-name",
-            startTime: "2021-01-01T00:00:00.000Z",
-            endTime: "2021-01-01T00:00:00.000Z",
-            input: { input: "value" },
-            metadata: { meta: "value" },
-            version: "2.0.0",
+          },
+        },
+        {
+          id: v4(),
+          type: "generation-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: generationId,
+            traceId: traceId,
+            parentObservationId: spanId,
+            modelParameters: { someKey: ["user-1", "user-2"] },
+          },
+        },
+        {
+          id: v4(),
+          type: "generation-update",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: generationId,
+            name: "generation-name",
+          },
+        },
+        {
+          id: v4(),
+          type: "event-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: eventId,
+            traceId: traceId,
+            name: "event-name",
+            parentObservationId: generationId,
           },
         },
         {
@@ -79,8 +627,17 @@ describe("/api/public/ingestion API Endpoint", () => {
           body: {
             id: scoreId,
             name: "score-name",
-            value: 100.5,
             traceId: traceId,
+            value: 100.5,
+            observationId: generationId,
+          },
+        },
+        {
+          id: v4(),
+          type: "sdk-log",
+          timestamp: new Date().toISOString(),
+          body: {
+            log: exception,
           },
         },
       ],
@@ -90,17 +647,23 @@ describe("/api/public/ingestion API Endpoint", () => {
 
     const dbTrace = await prisma.trace.findMany({
       where: {
-        name: "trace-name",
+        id: traceId,
       },
     });
 
     expect(dbTrace.length).toBeGreaterThan(0);
-    expect(dbTrace[0]?.name).toBe("trace-name");
-    expect(dbTrace[0]?.release).toBe("1.0.0");
-    expect(dbTrace[0]?.externalId).toBeNull();
-    expect(dbTrace[0]?.version).toBe("2.0.0");
     expect(dbTrace[0]?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
 
+    const dbSpan = await prisma.observation.findUnique({
+      where: {
+        id: spanId,
+      },
+    });
+
+    expect(dbSpan?.id).toBe(spanId);
+    expect(dbSpan?.name).toBe("span-name");
+    expect(dbSpan?.traceId).toBe(traceId);
+
     const dbGeneration = await prisma.observation.findUnique({
       where: {
         id: generationId,
@@ -110,34 +673,21 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbGeneration?.id).toBe(generationId);
     expect(dbGeneration?.traceId).toBe(traceId);
     expect(dbGeneration?.name).toBe("generation-name");
-    expect(dbGeneration?.startTime).toEqual(
-      new Date("2021-01-01T00:00:00.000Z"),
-    );
-    expect(dbGeneration?.endTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
-    expect(dbGeneration?.model).toBeNull();
-    expect(dbGeneration?.modelParameters).toEqual({ key: "value" });
-    expect(dbGeneration?.input).toEqual({ key: "value" });
-    expect(dbGeneration?.metadata).toEqual({ key: "value" });
-    expect(dbGeneration?.version).toBe("2.0.0");
-    expect(dbGeneration?.promptTokens).toEqual(400);
-    expect(dbGeneration?.completionTokens).toEqual(1000);
-    expect(dbGeneration?.output).toEqual({
-      key: "this is a great gpt output",
+    expect(dbGeneration?.parentObservationId).toBe(spanId);
+    expect(dbGeneration?.modelParameters).toEqual({
+      someKey: ["user-1", "user-2"],
     });
 
-    const dbSpan = await prisma.observation.findUnique({
+    const dbEvent = await prisma.observation.findUnique({
       where: {
-        id: spanId,
+        id: eventId,
       },
     });
 
-    expect(dbSpan?.id).toBe(spanId);
-    expect(dbSpan?.name).toBe("span-name");
-    expect(dbSpan?.startTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
-    expect(dbSpan?.endTime).toEqual(new Date("2021-01-:00:00.000Z"));
-    expect(dbSpan?.input).toEqual({ input: "value" });
-    expect(dbSpan?.metadata).toEqual({ meta: "value" });
-    expect(dbSpan?.version).toBe("2.0.0");
+    expect(dbEvent?.id).toBe(eventId);
+    expect(dbEvent?.traceId).toBe(traceId);
+    expect(dbEvent?.name).toBe("event-name");
+    expect(dbEvent?.parentObservationId).toBe(generationId);
 
     const dbScore = await prisma.score.findUnique({
       where: {
@@ -147,9 +697,21 @@ describe("/api/public/ingestion API Endpoint", () => {
 
     expect(dbScore?.id).toBe(scoreId);
     expect(dbScore?.traceId).toBe(traceId);
-    expect(dbScore?.name).toBe("score-name");
+    expect(dbScore?.observationId).toBe(generationId);
     expect(dbScore?.value).toBe(100.5);
-    expect(dbScore?.observationId).toBeNull();
+
+    const logEvent = await prisma.events.findFirst({
+      where: {
+        data: {
+          path: ["body", "log"],
+          string_contains: "ERROR",
+        },
+      },
+    });
+
+    expect(logEvent).toBeDefined();
+    expect(logEvent).not.toBeFalsy();
+    expect(JSON.stringify(logEvent?.data)).toContain("KeyError: 'model_name'");
   });
 
   it("should upsert threats", async () => {
@@ -353,6 +915,21 @@ describe("/api/public/ingestion API Endpoint", () => {
     const traceId = v4();
     const generationId = v4();
 
+    await prisma.model.create({
+      data: {
+        modelName: "gpt-3.5",
+        matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+        startDate: new Date("2021-01-01T00:00:00.000Z"),
+        unit: ModelUsageUnit.Tokens,
+        tokenizerId: "openai",
+        tokenizerConfig: {
+          tokensPerMessage: 3,
+          tokensPerName: 1,
+          tokenizerModel: "gpt-3.5-turbo",
+        },
+      },
+    });
+
     const responseOne = await makeAPICall("POST", "/api/public/ingestion", {
       batch: [
         {
@@ -420,6 +997,21 @@ describe("/api/public/ingestion API Endpoint", () => {
     const traceId = v4();
     const generationId = v4();
 
+    await prisma.model.create({
+      data: {
+        modelName: "gpt-3.5",
+        matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
+        startDate: new Date("2021-01-01T00:00:00.000Z"),
+        unit: ModelUsageUnit.Tokens,
+        tokenizerId: "openai",
+        tokenizerConfig: {
+          tokensPerMessage: 3,
+          tokensPerName: 1,
+          tokenizerModel: "gpt-3.5-turbo",
+        },
+      },
+    });
+
     const responseOne = await makeAPICall("POST", "/api/public/ingestion", {
       batch: [
         {
@@ -529,6 +1121,66 @@ describe("/api/public/ingestion API Endpoint", () => {
     expect(dbTrace[0]?.version).toBe("2.0.0");
   });
 
+  it("should not override a trace from a different project", async () => {
+    const traceId = v4();
+    const newProjectId = v4();
+
+    await prisma.project.create({
+      data: {
+        id: newProjectId,
+        name: "another-project",
+      },
+    });
+
+    await prisma.trace.create({
+      data: {
+        id: traceId,
+        project: { connect: { id: newProjectId } },
+      },
+    });
+
+    const responseOne = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name: "trace-name",
+            userId: "user-1",
+            metadata: { key: "value" },
+            release: "1.0.0",
+            version: "2.0.0",
+          },
+        },
+      ],
+    });
+    expect(responseOne.status).toBe(207);
+
+    console.log(responseOne.body);
+
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
+    const errors = responseOne.body.errors;
+
+    expect(errors).toBeDefined();
+    console.log(errors);
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    expect(errors.length).toBe(1);
+
+    const dbTrace = await prisma.trace.findMany({
+      where: {
+        id: traceId,
+      },
+    });
+
+    expect(dbTrace.length).toEqual(1);
+    expect(dbTrace[0]?.name).toBeNull();
+    expect(dbTrace[0]?.release).toBeNull();
+    expect(dbTrace[0]?.metadata).toBeNull();
+    expect(dbTrace[0]?.version).toBeNull();
+  });
+
   [
     {
       inputs: [{ a: "a" }, { b: "b" }],
@@ -686,4 +1338,80 @@ describe("/api/public/ingestion API Endpoint", () => {
 
     expect(dbGeneration).toBeTruthy();
   });
+
+  it("filters out NULL characters", async () => {
+    const traceId = v4();
+    const generationId = v4();
+
+    const responseOne = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name: "trace-name",
+          },
+        },
+        {
+          id: v4(),
+          type: "observation-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: generationId,
+            traceId: traceId,
+            type: "GENERATION",
+            name: "generation-name",
+            traceIdType: "LANGFUSE",
+            input: {
+              key: "IB\nibo.org Change site\nIB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an opportunity for schools  IB Home   /   News   /   News about the IB   /   Why ChatGPT is an opportunity for schools  Why ChatGPT is an opportunity for  schools  Published:   06 March 2023   Last updated:   06 June 2023  Date published:   28 February 2023  Dr Matthew Glanville, Head of Assessment Principles and Practice  Source:   Why ChatGPT is an opportunity for schools | The Times  Those of us who work in the schools or exam sector should not be terri \u0000 ed by ChatGPT and  the rise of AI software – we should be excited. We should embrace it as an extraordinary  opportunity.  Contrary to some stark warnings, it is not the end of exams, nor even a huge threat to  coursework, but it does bring into very sharp focus the impact that arti \u0000 ",
+            },
+            output: {
+              key: "제점이 있었죠. 그중 하나는 일제가 한국의 신용체계를 망가뜨린 채 한국을 떠났다는 겁니다. 해방전 일제는 조선의 신용체계를 거의 독점적으로 소유한 상황이었습니다. 1945년 6월 기준 일제는 조선의 본점을 둔 전체은행 5개의 불입자본 총액의 89.7%를",
+            },
+          },
+        },
+      ],
+    });
+    expect(responseOne.status).toBe(207);
+
+    const dbTrace = await prisma.trace.findMany({
+      where: {
+        id: traceId,
+      },
+    });
+
+    expect(dbTrace.length).toEqual(1);
+
+    const dbGeneration = await prisma.observation.findUnique({
+      where: {
+        id: generationId,
+      },
+    });
+    expect(dbGeneration?.input).toStrictEqual({
+      key: `IB
+ibo.org Change site
+IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an opportunity for schools  IB Home   /   News   /   News about the IB   /   Why ChatGPT is an opportunity for schools  Why ChatGPT is an opportunity for  schools  Published:   06 March 2023   Last updated:   06 June 2023  Date published:   28 February 2023  Dr Matthew Glanville, Head of Assessment Principles and Practice  Source:   Why ChatGPT is an opportunity for schools | The Times  Those of us who work in the schools or exam sector should not be terri  ed by ChatGPT and  the rise of AI software – we should be excited. We should embrace it as an extraordinary  opportunity.  Contrary to some stark warnings, it is not the end of exams, nor even a huge threat to  coursework, but it does bring into very sharp focus the impact that arti  `,
+    });
+    expect(dbGeneration?.output).toStrictEqual({
+      key: "제점이 있었죠. 그중 하나는 일제가 한국의 신용체계를 망가뜨린 채 한국을 떠났다는 겁니다. 해방전 일제는 조선의 신용체계를 거의 독점적으로 소유한 상황이었습니다. 1945년 6월 기준 일제는 조선의 본점을 둔 전체은행 5개의 불입자본 총액의 89.7%를",
+    });
+
+    expect(dbGeneration).toBeTruthy();
+  });
+
+  [
+    { input: "A\u0000hallo", expected: "Ahallo" },
+    { input: ["A\u0000hallo"], expected: ["Ahallo"] },
+    { input: { obj: ["A\u0000hallo"] }, expected: { obj: ["Ahallo"] } },
+  ].forEach(({ input, expected }) => {
+    it(`cleans events with null values ${JSON.stringify(
+      input,
+    )} ${JSON.stringify(expected)}`, () => {
+      const cleanedEvent = cleanEvent(input);
+      console.log(cleanedEvent);
+      expect(cleanedEvent).toStrictEqual(expected);
+    });
+  });
 });

src/__tests__/model-match.servertest.ts

@@ -0,0 +1,96 @@
+/** @jest-environment node */
+
+import { modelMatch } from "@/scripts/model-match";
+import { pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
+import { prisma } from "@/src/server/db";
+
+describe("model match", () => {
+  beforeEach(async () => await pruneDatabase());
+
+  it("should match historic observations to models", async () => {
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000010",
+        outputPrice: "0.0000020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-3.5-turbo)?(.*)",
+        projectId: null,
+        unit: ModelUsageUnit.Tokens,
+        tokenizerConfig: {
+          tokensPerMessage: 3,
+          tokensPerName: 1,
+          tokenizerModel: "gpt-3.5-turbo",
+        },
+        tokenizerId: "openai",
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "claude-1.3",
+        inputPrice: "0.0000010",
+        outputPrice: "0.0000020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(claude-1.3)?(.*)",
+        projectId: null,
+        unit: ModelUsageUnit.Tokens,
+        tokenizerId: "claude",
+      },
+    });
+
+    await prisma.observation.createMany({
+      data: [
+        {
+          id: "observation-1",
+          type: "GENERATION",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          model: "gpt-3.5-turbo",
+          startTime: new Date("2024-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          promptTokens: 200,
+          completionTokens: 3000,
+          input: "I am a prompt",
+          output: "I am a completion",
+        },
+        {
+          type: "GENERATION",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          model: "claude-1.3",
+          startTime: new Date("2024-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          input: "I am a prompt",
+          output: "I am a completion",
+        },
+        {
+          type: "GENERATION",
+          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          model: "claude-1.3",
+          startTime: new Date("2024-01-01T00:00:00.000Z"),
+          unit: ModelUsageUnit.Tokens,
+          input: "I am a prompt",
+          output: "I am a completion",
+        },
+      ],
+    });
+
+    await modelMatch();
+
+    const observations = await prisma.observation.findMany();
+
+    // check that tokens from observation-1 are not changed
+    const observation1 = observations.find((o) => o.id === "observation-1");
+    expect(observation1?.promptTokens).toEqual(200);
+    expect(observation1?.completionTokens).toEqual(3000);
+    expect(observation1?.totalTokens).toEqual(0);
+
+    expect(observations.length).toEqual(3);
+    observations.forEach((observation) => {
+      expect(observation.internalModel).toBeDefined();
+      expect(observation.promptTokens).toBeGreaterThan(0);
+      expect(observation.completionTokens).toBeGreaterThan(0);
+    });
+  });
+});

src/__tests__/observations.servertest.ts

@@ -1,11 +1,13 @@
 /** @jest-environment node */
 
-import { prisma } from "@/src/server/db";
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
-import { type Observation } from "@prisma/client";
 
-describe("/api/public/generations API Endpoint", () => {
+import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
+import { prisma } from "@/src/server/db";
+import { type ObservationView } from "@prisma/client";
+
+describe("/api/public/observations API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
   afterEach(async () => await pruneDatabase());
 
@@ -26,6 +28,19 @@ describe("/api/public/generations API Endpoint", () => {
       },
     });
 
+    const model = await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000010",
+        outputPrice: "0.0000020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        projectId: null,
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+
     await prisma.observation.create({
       data: {
         id: uuidv4(),
@@ -33,15 +48,20 @@ describe("/api/public/generations API Endpoint", () => {
         name: "generation-name",
         startTime: new Date("2021-01-01T00:00:00.000Z"),
         endTime: new Date("2021-01-01T00:00:00.000Z"),
-        model: "model-name",
+        model: "gpt-3.5-turbo",
         modelParameters: { key: "value" },
         input: { key: "input" },
         output: { key: "output" },
+        promptTokens: 10,
+        completionTokens: 20,
+        totalTokens: 30,
         version: "2.0.0",
         type: "GENERATION",
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
         },
+        internalModel: "gpt-3.5-turbo",
+        unit: ModelUsageUnit.Tokens,
       },
     });
 
@@ -54,13 +74,27 @@ describe("/api/public/generations API Endpoint", () => {
     expect(fetchedObservations.status).toBe(200);
 
     if (!isObservationList(fetchedObservations.body)) {
-      throw new Error("Expected body to be an array of observations");
+      throw new Error(
+        "Expected body to be an array of observations" +
+          JSON.stringify(fetchedObservations.body),
+      );
     }
 
     expect(fetchedObservations.body.data.length).toBe(1);
     expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
     expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
     expect(fetchedObservations.body.data[0]?.output).toEqual({ key: "output" });
+    expect(fetchedObservations.body.data[0]?.model).toEqual("gpt-3.5-turbo");
+    expect(fetchedObservations.body.data[0]?.modelId).toEqual(model.id);
+    expect(
+      fetchedObservations.body.data[0]?.calculatedInputCost,
+    ).toBeGreaterThan(0);
+    expect(
+      fetchedObservations.body.data[0]?.calculatedOutputCost,
+    ).toBeGreaterThan(0);
+    expect(
+      fetchedObservations.body.data[0]?.calculatedTotalCost,
+    ).toBeGreaterThan(0);
   });
   it("should fetch all observations, filtered by generations", async () => {
     await pruneDatabase();
@@ -79,6 +113,19 @@ describe("/api/public/generations API Endpoint", () => {
       },
     });
 
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000010",
+        outputPrice: "0.0000020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        projectId: null,
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+
     await prisma.observation.create({
       data: {
         id: uuidv4(),
@@ -86,11 +133,16 @@ describe("/api/public/generations API Endpoint", () => {
         name: "generation-name",
         startTime: new Date("2021-01-01T00:00:00.000Z"),
         endTime: new Date("2021-01-01T00:00:00.000Z"),
-        model: "model-name",
+        model: "gpt-3.5-turbo",
+        internalModel: "gpt-3.5-turbo",
         modelParameters: { key: "value" },
         input: { key: "input" },
         output: { key: "output" },
+        promptTokens: 10,
+        completionTokens: 20,
+        totalTokens: 30,
         version: "2.0.0",
+        unit: ModelUsageUnit.Tokens,
         type: "GENERATION",
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
@@ -105,7 +157,6 @@ describe("/api/public/generations API Endpoint", () => {
         name: "generation-name",
         startTime: new Date("2021-01-01T00:00:00.000Z"),
         endTime: new Date("2021-01-01T00:00:00.000Z"),
-        model: "model-name",
         modelParameters: { key: "value" },
         input: { key: "input" },
         output: { key: "output" },
@@ -123,6 +174,8 @@ describe("/api/public/generations API Endpoint", () => {
       undefined,
     );
 
+    console.log(fetchedObservations.body);
+
     expect(fetchedObservations.status).toBe(200);
 
     if (!isObservationList(fetchedObservations.body)) {
@@ -156,11 +209,18 @@ const isObservationList = (val: unknown): val is ObservationResponse => {
         "input" in element &&
         "output" in element &&
         "metadata" in element &&
-        "version" in element,
+        "version" in element &&
+        "modelId" in element &&
+        "inputPrice" in element &&
+        "outputPrice" in element &&
+        "totalPrice" in element &&
+        "calculatedInputCost" in element &&
+        "calculatedOutputCost" in element &&
+        "calculatedTotalCost" in element,
     )
   );
 };
 
 type ObservationResponse = {
-  data: Observation[];
+  data: ObservationView[];
 };

src/__tests__/prompts.servertest.ts

@@ -0,0 +1,344 @@
+/** @jest-environment node */
+
+import { prisma } from "@/src/server/db";
+import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { v4 as uuidv4, v4 } from "uuid";
+import { type Prompt } from "@prisma/client";
+
+describe("/api/public/prompts API Endpoint", () => {
+  beforeEach(async () => await pruneDatabase());
+
+  it("should fetch a prompt", async () => {
+    const promptId = uuidv4();
+
+    await prisma.prompt.create({
+      data: {
+        id: promptId,
+        name: "prompt-name",
+        prompt: "prompt",
+        isActive: true,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
+    const fetchedObservations = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=1",
+      undefined,
+    );
+
+    expect(fetchedObservations.status).toBe(200);
+
+    if (!isPrompt(fetchedObservations.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+
+    expect(fetchedObservations.body.id).toBe(promptId);
+    expect(fetchedObservations.body.name).toBe("prompt-name");
+    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.version).toBe(1);
+    expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.createdBy).toBe("user-1");
+  });
+
+  it("should fetch active prompt only if no prompt version is given", async () => {
+    const promptId = uuidv4();
+
+    await prisma.prompt.create({
+      data: {
+        id: promptId,
+        name: "prompt-name",
+        prompt: "prompt",
+        isActive: false,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
+    const fetchedObservations = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name",
+      undefined,
+    );
+
+    expect(fetchedObservations.status).toBe(404);
+  });
+
+  it("should fetch inactive prompt if prompt version is given", async () => {
+    const promptId = uuidv4();
+    const promptTwoId = uuidv4();
+
+    await prisma.prompt.create({
+      data: {
+        id: promptId,
+        name: "prompt-name",
+        prompt: "prompt-one",
+        isActive: false,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
+    await prisma.prompt.create({
+      data: {
+        id: promptTwoId,
+        name: "prompt-name",
+        prompt: "prompt",
+        isActive: true,
+        version: 2,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
+    const fetchedObservations = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=1",
+      undefined,
+    );
+
+    expect(fetchedObservations.status).toBe(200);
+
+    if (!isPrompt(fetchedObservations.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+
+    expect(fetchedObservations.body.id).toBe(promptId);
+    expect(fetchedObservations.body.name).toBe("prompt-name");
+    expect(fetchedObservations.body.prompt).toBe("prompt-one");
+    expect(fetchedObservations.body.version).toBe(1);
+    expect(fetchedObservations.body.isActive).toBe(false);
+    expect(fetchedObservations.body.createdBy).toBe("user-1");
+  });
+
+  it("should fetch active prompt when multiple exist", async () => {
+    const promptIdOne = uuidv4();
+    const promptIdTwo = uuidv4();
+
+    await prisma.prompt.create({
+      data: {
+        id: promptIdOne,
+        name: "prompt-name",
+        prompt: "prompt",
+        isActive: false,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
+    await prisma.prompt.create({
+      data: {
+        id: promptIdTwo,
+        name: "prompt-name",
+        prompt: "prompt",
+        isActive: true,
+        version: 2,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
+    const fetchedObservations = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name",
+      undefined,
+    );
+
+    expect(fetchedObservations.status).toBe(200);
+
+    if (!isPrompt(fetchedObservations.body)) {
+      throw new Error("Expected body to be a prompt");
+    }
+
+    expect(fetchedObservations.body.id).toBe(promptIdTwo);
+    expect(fetchedObservations.body.name).toBe("prompt-name");
+    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.version).toBe(2);
+    expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.createdBy).toBe("user-1");
+  });
+
+  it("should create and fetch a prompt", async () => {
+    await makeAPICall("POST", "/api/public/prompts", {
+      name: "prompt-name",
+      prompt: "prompt",
+      isActive: true,
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+
+    const fetchedObservations = await makeAPICall(
+      "GET",
+      "/api/public/prompts?name=prompt-name&version=1",
+      undefined,
+    );
+
+    expect(fetchedObservations.status).toBe(200);
+
+    if (!isPrompt(fetchedObservations.body)) {
+      throw new Error("Expected body to be an array of observations");
+    }
+
+    expect(fetchedObservations.body.name).toBe("prompt-name");
+    expect(fetchedObservations.body.prompt).toBe("prompt");
+    expect(fetchedObservations.body.version).toBe(1);
+    expect(fetchedObservations.body.isActive).toBe(true);
+    expect(fetchedObservations.body.createdBy).toBe("API");
+  });
+
+  it("should relate generation to prompt", async () => {
+    const traceId = v4();
+    const generationId = v4();
+
+    const promptId = uuidv4();
+
+    await prisma.prompt.create({
+      data: {
+        id: promptId,
+        name: "prompt-name",
+        prompt: "prompt",
+        isActive: true,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      metadata: {
+        sdk_verion: "1.0.0",
+        sdk_name: "python",
+      },
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name: "trace-name",
+          },
+        },
+        {
+          id: v4(),
+          type: "generation-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: generationId,
+            traceId: traceId,
+            type: "GENERATION",
+            name: "generation-name",
+            promptName: "prompt-name",
+            promptVersion: 1,
+          },
+        },
+      ],
+    });
+
+    expect(response.status).toBe(207);
+
+    const dbGeneration = await prisma.observation.findUnique({
+      where: {
+        id: generationId,
+      },
+    });
+
+    expect(dbGeneration?.id).toBe(generationId);
+    expect(dbGeneration?.promptId).toBe(promptId);
+  });
+
+  it("should fail if prompt version is missing", async () => {
+    const traceId = v4();
+    const generationId = v4();
+
+    const promptId = uuidv4();
+
+    await prisma.prompt.create({
+      data: {
+        id: promptId,
+        name: "prompt-name",
+        prompt: "prompt",
+        isActive: true,
+        version: 1,
+        project: {
+          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+        },
+        createdBy: "user-1",
+      },
+    });
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      metadata: {
+        sdk_verion: "1.0.0",
+        sdk_name: "python",
+      },
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name: "trace-name",
+          },
+        },
+        {
+          id: v4(),
+          type: "generation-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: generationId,
+            traceId: traceId,
+            type: "GENERATION",
+            name: "generation-name",
+            promptName: "prompt-name",
+          },
+        },
+      ],
+    });
+
+    expect(response.status).toBe(207);
+
+    console.log("response body", response.body);
+
+    const dbGeneration = await prisma.observation.findUnique({
+      where: {
+        id: generationId,
+      },
+    });
+
+    expect(dbGeneration).toBeNull();
+  });
+});
+
+const isPrompt = (x: unknown): x is Prompt => {
+  if (typeof x !== "object" || x === null) return false;
+  const prompt = x as Prompt;
+  return (
+    typeof prompt.id === "string" &&
+    typeof prompt.name === "string" &&
+    typeof prompt.version === "number" &&
+    typeof prompt.prompt === "string" &&
+    typeof prompt.isActive === "boolean" &&
+    typeof prompt.projectId === "string" &&
+    typeof prompt.createdBy === "string"
+  );
+};

src/__tests__/query-builder.servertest.ts

@@ -22,7 +22,7 @@ describe("Build valid SQL queries", () => {
         table: "traces_observations",
         values: ["project-id", "project-id"],
         strings: [
-          ' FROM  traces t LEFT JOIN observations o ON t.id = o.trace_id  WHERE  t."project_id" = ',
+          ' FROM  traces t LEFT JOIN observations_view o ON t.id = o.trace_id  WHERE  t."project_id" = ',
           '  AND o."project_id" = ',
           " ;",
         ],
@@ -30,7 +30,7 @@ describe("Build valid SQL queries", () => {
       {
         table: "observations",
         values: ["project-id"],
-        strings: [' FROM  observations o  WHERE  o."project_id" = ', " ;"],
+        strings: [' FROM  observations_view o  WHERE  o."project_id" = ', " ;"],
       } as const,
       {
         table: "traces_scores",

src/__tests__/server/api/routers/traces.servertest.ts

@@ -0,0 +1,107 @@
+/** @jest-environment node */
+import { pruneDatabase } from "@/src/__tests__/test-utils";
+import { appRouter } from "@/src/server/api/root";
+import { createInnerTRPCContext } from "@/src/server/api/trpc";
+import { prisma } from "@/src/server/db";
+import type { Session } from "next-auth";
+
+describe("Traces TRPC Router", () => {
+  beforeEach(async () => await pruneDatabase());
+  afterEach(async () => await pruneDatabase());
+
+  const session: Session = {
+    expires: "1",
+    user: {
+      id: "clgb17vnp000008jjere5g15i",
+      name: "John Doe",
+      projects: [
+        {
+          id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          role: "ADMIN",
+          name: "test",
+        },
+      ],
+      featureFlags: {
+        templateFlag: true,
+      },
+      admin: true,
+    },
+  };
+
+  const ctx = createInnerTRPCContext({ session });
+  const caller = appRouter.createCaller({ ...ctx, prisma });
+
+  test("traces.all RPC returns an array of traces", async () => {
+    const trace = {
+      name: "trace-name",
+      userId: "user-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "2.0.0",
+    };
+    await prisma.trace.create({
+      data: { ...trace, projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+    });
+
+    const traces = await caller.traces.all({
+      page: 0,
+      limit: 10,
+      // projectId from `seed.ts`
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      filter: null,
+      searchQuery: "",
+      orderBy: null,
+    });
+    expect(traces).toBeDefined();
+    expect(traces).toMatchObject({ traces: [trace] });
+  });
+
+  test("traces.all RPC orders traces by userId", async () => {
+    const traceTmpl = {
+      name: "trace-name",
+      userId: "user-1",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "2.0.0",
+    };
+    const trace1 = traceTmpl;
+    const trace2 = { ...traceTmpl, userId: "user-2" };
+    await prisma.trace.create({
+      data: { ...traceTmpl, projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+    });
+    await prisma.trace.create({
+      data: {
+        ...trace2,
+        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      },
+    });
+
+    const tracesASC = await caller.traces.all({
+      page: 0,
+      limit: 10,
+      // projectId from `seed.ts`
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      filter: null,
+      searchQuery: "",
+      orderBy: {
+        column: "userId",
+        order: "ASC",
+      },
+    });
+    expect(tracesASC).toMatchObject({ traces: [trace1, trace2] });
+
+    const tracesDESC = await caller.traces.all({
+      page: 0,
+      limit: 10,
+      // projectId from `seed.ts`
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      filter: null,
+      searchQuery: "",
+      orderBy: {
+        column: "userId",
+        order: "DESC",
+      },
+    });
+    expect(tracesDESC).toMatchObject({ traces: [trace2, trace1] });
+  });
+});

src/__tests__/sessions.servertest.ts

@@ -0,0 +1,62 @@
+/** @jest-environment node */
+
+import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { prisma } from "@/src/server/db";
+
+describe("/api/public/traces API Endpoint", () => {
+  beforeEach(async () => await pruneDatabase());
+  afterEach(async () => await pruneDatabase());
+
+  it("should create a session via a trace", async () => {
+    await pruneDatabase();
+
+    await makeAPICall("POST", "/api/public/traces", {
+      name: "trace-name",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      sessionId: "session-id",
+    });
+
+    const dbSession = await prisma.traceSession.findFirst({
+      where: {
+        id: "session-id",
+      },
+    });
+
+    expect(dbSession).not.toBeNull();
+    expect(dbSession).toMatchObject({
+      id: "session-id",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+    });
+  });
+
+  it("should get session including traces", async () => {
+    await pruneDatabase();
+
+    await makeAPICall("POST", "/api/public/traces", {
+      name: "trace-name",
+      id: "trace-id",
+      input: { hello: "world" },
+      output: "hi",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      sessionId: "session-id",
+    });
+
+    const response = await makeAPICall(
+      "GET",
+      "/api/public/sessions/session-id",
+    );
+
+    expect(response.status).toBe(200);
+    expect(response.body).toMatchObject({
+      id: "session-id",
+      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      traces: [
+        {
+          id: "trace-id",
+          input: { hello: "world" },
+          output: "hi",
+        },
+      ],
+    });
+  });
+});

src/__tests__/spans.servertest.ts

@@ -140,8 +140,8 @@ describe("/api/public/spans API Endpoint", () => {
         name: "spanName",
       },
     });
-    expect(dbSpan?.id).toBe(spanId);
-    expect(dbSpan?.traceId).toBe(traceId);
+    expect(dbSpan.id).toBe(spanId);
+    expect(dbSpan.traceId).toBe(traceId);
   });
 
   it("should create trace when creating span without existing trace", async () => {
@@ -248,14 +248,14 @@ describe("/api/public/spans API Endpoint", () => {
 
     expect(createSpan.status).toBe(200);
 
-    expect(dbSpan?.id).toBe(spanId);
-    expect(dbSpan?.traceId).toBe(dbTrace[0]?.id);
-    expect(dbSpan?.name).toBe(generationName);
-    expect(dbSpan?.startTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
-    expect(dbSpan?.endTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
-    expect(dbSpan?.input).toEqual({ key: "value" });
-    expect(dbSpan?.metadata).toEqual({ key: "value" });
-    expect(dbSpan?.version).toBe("2.0.0");
+    expect(dbSpan.id).toBe(spanId);
+    expect(dbSpan.traceId).toBe(dbTrace[0]?.id);
+    expect(dbSpan.name).toBe(generationName);
+    expect(dbSpan.startTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
+    expect(dbSpan.endTime).toEqual(new Date("2021-01-01T00:00:00.000Z"));
+    expect(dbSpan.input).toEqual({ key: "value" });
+    expect(dbSpan.metadata).toEqual({ key: "value" });
+    expect(dbSpan.version).toBe("2.0.0");
   });
 
   it("should update span", async () => {

src/__tests__/test-utils.ts

@@ -9,6 +9,12 @@ export const pruneDatabase = async () => {
   await prisma.score.deleteMany();
   await prisma.observation.deleteMany();
   await prisma.trace.deleteMany();
+  await prisma.datasetItem.deleteMany();
+  await prisma.dataset.deleteMany();
+  await prisma.datasetRuns.deleteMany();
+  await prisma.prompt.deleteMany();
+  await prisma.events.deleteMany();
+  await prisma.model.deleteMany();
 };
 
 export function createBasicAuthHeader(
@@ -24,7 +30,7 @@ export function createBasicAuthHeader(
 export async function makeAPICall(
   method: "POST" | "GET" | "PUT" | "DELETE" | "PATCH",
   url: string,
-  body: unknown,
+  body?: unknown,
 ) {
   const finalUrl = `http://localhost:3000/${url}`;
   const options = {
@@ -37,7 +43,9 @@ export async function makeAPICall(
         "sk-lf-1234567890",
       ),
     },
-    body: JSON.stringify(body),
+    // Conditionally include the body property if the method is not "GET"
+    ...(method !== "GET" &&
+      body !== undefined && { body: JSON.stringify(body) }),
   };
   const a = await fetch(finalUrl, options);
 

src/__tests__/traces.servertest.ts

@@ -40,33 +40,100 @@ describe("/api/public/traces API Endpoint", () => {
       id: "trace-id",
       name: "trace-name",
       userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
       metadata: { key: "value" },
       release: "1.0.0",
       version: "2.0.0",
+      public: true,
+    });
+
+    const dbTrace1 = await prisma.trace.findFirst({
+      where: {
+        id: "trace-id",
+      },
+    });
+
+    expect(dbTrace1).not.toBeNull();
+    expect(dbTrace1).toMatchObject({
+      name: "trace-name",
+      release: "1.0.0",
+      externalId: null,
+      version: "2.0.0",
+      public: true,
+      userId: "user-1",
     });
 
     await makeAPICall("POST", "/api/public/traces", {
       id: "trace-id",
-      name: "trace-name",
-      userId: "user-1",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
       metadata: { key: "value" },
       release: "1.0.0",
       version: "5.0.0",
+      public: false,
     });
 
-    const dbTrace = await prisma.trace.findMany({
+    const dbTrace2 = await prisma.trace.findFirst({
       where: {
         name: "trace-name",
       },
     });
 
-    expect(dbTrace.length).toBeGreaterThan(0);
-    expect(dbTrace[0]?.name).toBe("trace-name");
-    expect(dbTrace[0]?.release).toBe("1.0.0");
-    expect(dbTrace[0]?.externalId).toBeNull();
-    expect(dbTrace[0]?.version).toBe("5.0.0");
-    expect(dbTrace[0]?.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+    expect(dbTrace2).not.toBeNull();
+    expect(dbTrace2).toMatchObject({
+      name: "trace-name",
+      release: "1.0.0",
+      externalId: null,
+      version: "5.0.0",
+      public: false,
+      userId: "user-1",
+    });
+  });
+
+  it("should use tags correctly on POST and GET", async () => {
+    await pruneDatabase();
+
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-1",
+      tags: ["tag-1", "tag-2", "tag-3"],
+    });
+
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-2",
+      tags: ["tag-1"],
+    });
+
+    await makeAPICall("POST", "/api/public/traces", {
+      id: "trace-3",
+      tags: ["tag-2", "tag-3"],
+    });
+
+    // multiple tags
+    const traces = await makeAPICall(
+      "GET",
+      "/api/public/traces?tags=tag-2&tags=tag-3",
+    );
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+    const traceIds = traces.body.data.map((t: { id: string }) => t.id);
+    // check for equality ok as ordered by timestamp
+    expect(traceIds).toEqual(["trace-3", "trace-1"]);
+
+    // single tag
+    const traces2 = await makeAPICall("GET", "/api/public/traces?tags=tag-1");
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+    const traceIds2 = traces2.body.data.map((t: { id: string }) => t.id);
+    // check for equality ok as ordered by timestamp
+    expect(traceIds2).toEqual(["trace-2", "trace-1"]);
+
+    // wrong tag
+    const traces3 = await makeAPICall("GET", "/api/public/traces?tags=tag-10");
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+    const traceIds3 = traces3.body.data.map((t: { id: string }) => t.id);
+    // check for equality ok as ordered by timestamp
+    expect(traceIds3).toEqual([]);
+
+    // no tag
+    const traces4 = await makeAPICall("GET", "/api/public/traces?tags=");
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+    const traceIds4 = traces4.body.data.map((t: { id: string }) => t.id);
+    // check for equality ok as ordered by timestamp
+    expect(traceIds4).toEqual(["trace-3", "trace-2", "trace-1"]);
   });
 });

src/__tests__/usage.servertest.ts

@@ -1,17 +1,42 @@
+import { ModelUsageUnit } from "@/src/constants";
 import { tokenCount } from "@/src/features/ingest/lib/usage";
 
 describe("Token Count Functions", () => {
+  const generateModel = (model: string, tokenizer: string) => {
+    return {
+      id: "1",
+      modelName: model,
+      tokenizerId: tokenizer,
+      tokenizerConfig: {
+        tokensPerMessage: 3,
+        tokensPerName: 1,
+        tokenizerModel: model,
+      },
+      createdAt: new Date(),
+      updatedAt: new Date(),
+      matchPattern: "",
+      projectId: null,
+      startDate: null,
+      inputPrice: null,
+      outputPrice: null,
+      totalPrice: null,
+      unit: ModelUsageUnit.Tokens,
+    };
+  };
+
   describe("token count for strings", () => {
     [
-      { model: "gpt-3.5", tokens: 114 },
-      { model: "gpt-4-1106-preview", tokens: 114 },
-      { model: "gpt-4-vision-preview", tokens: 114 },
-      { model: "claude", tokens: 118 },
-      { model: "claude-instant-1.2", tokens: 118 },
-    ].forEach(({ model, tokens }) => {
+      { model: "gpt-3.5-turbo", tokenizer: "openai", tokens: 114 },
+      { model: "text-embedding-ada-002", tokenizer: "openai", tokens: 114 },
+      { model: "gpt-4-1106-preview", tokenizer: "openai", tokens: 114 },
+      { model: "gpt-4-vision-preview", tokenizer: "openai", tokens: 114 },
+      { model: "claude", tokenizer: "claude", tokens: 118 },
+      { model: "claude-instant-1.2", tokenizer: "claude", tokens: 118 },
+      { model: "gpt-3.5-turbo-1106", tokenizer: "openai", tokens: 114 },
+    ].forEach(({ model, tokens, tokenizer }) => {
       it(`should return token count ${tokens} for ${model}`, () => {
         const result = tokenCount({
-          model: model,
+          model: generateModel(model, tokenizer),
           text: "Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.",
         });
         expect(result).toBeDefined();
@@ -21,20 +46,67 @@ describe("Token Count Functions", () => {
 
     it("should return undefined for unknown model", () => {
       const result = tokenCount({
-        model: "unknown-model",
+        model: generateModel("unknown-model", "unknown-tokenizer"),
         text: "Hello, World!",
       });
       expect(result).toBeUndefined();
     });
 
+    it("check extensive openai chat message", () => {
+      const result = tokenCount({
+        model: generateModel("gpt-3.5-turbo", "openai"),
+        text: [
+          {
+            role: "system",
+            content: "some test",
+            id: "some-id",
+            isPersisted: true,
+          },
+          {
+            id: "some-id",
+            content: "some test",
+            role: "user",
+            timestamp: "2024-01-00:00:00.488Z",
+            isPersisted: true,
+          },
+          {
+            id: "some id",
+            content:
+              "Hey Simon! 😊 How's your day going? Have you been up to anything interesting lately?",
+            role: "user",
+            timestamp: "2024-01-24T10:00:00.929Z",
+            isPersisted: true,
+          },
+          {
+            content: true,
+            role: "user",
+            id: "some id",
+          },
+          {
+            role: "system",
+            content: "This is some content",
+          },
+          {
+            id: "another id",
+            role: "assistant",
+            content: "This is some content",
+          },
+        ],
+      });
+      expect(result).toBe(155);
+    });
+
     it("should return for invalid text type", () => {
-      const result = tokenCount({ model: "gpt-4", text: 1234 });
+      const result = tokenCount({
+        model: generateModel("gpt-4", "openai"),
+        text: 1234,
+      });
       expect(result).toBe(2);
     });
 
     it("should return correct token count for empty string", () => {
       const result = tokenCount({
-        model: "gpt-4",
+        model: generateModel("gpt-4", "openai"),
         text: "",
       });
       expect(result).toBe(0);
@@ -43,7 +115,7 @@ describe("Token Count Functions", () => {
     it("should return correct token count for very long string", () => {
       const longString = "A".repeat(10000);
       const result = tokenCount({
-        model: "gpt-4",
+        model: generateModel("gpt-4", "openai"),
         text: longString,
       });
       expect(result).toBeDefined();
@@ -52,14 +124,14 @@ describe("Token Count Functions", () => {
 
     it("should return undefined for null text input", () => {
       const result = tokenCount({
-        model: "gpt-4",
+        model: generateModel("gpt-4", "openai"),
         text: null,
       });
       expect(result).toBeUndefined();
     });
     it("should return undefined for undefined text input", () => {
       const result = tokenCount({
-        model: "gpt-4",
+        model: generateModel("gpt-4", "openai"),
         text: undefined,
       });
       expect(result).toBeUndefined();
@@ -68,13 +140,14 @@ describe("Token Count Functions", () => {
 
   describe("token count for chat messages", () => {
     [
-      { model: "gpt-4", tokens: 44 },
-      { model: "gpt-3.5-turbo-16k-0613", tokens: 44 },
-      { model: "claude-instant-1.2", tokens: 48 },
-    ].forEach(({ model, tokens }) => {
+      { model: "gpt-4", tokenizer: "openai", tokens: 44 },
+      { model: "gpt-3.5-turbo-16k-0613", tokenizer: "openai", tokens: 44 },
+      { model: "gpt-3.5-turbo-16k-0613", tokenizer: "openai", tokens: 44 },
+      { model: "claude-instant-1.2", tokenizer: "claude", tokens: 48 },
+    ].forEach(({ model, tokens, tokenizer }) => {
       it(`should return token count ${tokens} for ${model}`, () => {
         const result = tokenCount({
-          model: model,
+          model: generateModel(model, tokenizer),
           text: [
             { role: "system", content: "You are a helpful assistant." },
             { role: "user", content: "Who won the world series in 2020?" },
@@ -91,7 +164,7 @@ describe("Token Count Functions", () => {
 
     it("should return for non array", () => {
       const result = tokenCount({
-        model: "gpt-4",
+        model: generateModel("gpt-4", "openai"),
         text: { role: "Helo world" },
       });
       expect(result).toBe(7);
@@ -99,7 +172,7 @@ describe("Token Count Functions", () => {
 
     it("should return for empty array", () => {
       const result = tokenCount({
-        model: "gpt-4",
+        model: generateModel("gpt-4", "openai"),
         text: [],
       });
       expect(result).toBeUndefined();
@@ -107,7 +180,7 @@ describe("Token Count Functions", () => {
 
     it("should return for array of invalid object", () => {
       const result = tokenCount({
-        model: "gpt-4",
+        model: generateModel("gpt-4", "openai"),
         text: [{ role: "Helo world" }],
       });
       expect(result).toBe(9);

src/components/LangfuseLogo.tsx

@@ -1,3 +1,7 @@
+import { AlertTriangle } from "lucide-react";
+
+import { VERSION } from "@/src/constants";
+import { env } from "@/src/env.mjs";
 import { cn } from "@/src/utils/tailwind";
 
 export const LangfuseIcon = ({
@@ -20,19 +24,55 @@ export const LangfuseIcon = ({
 export const LangfuseLogo = ({
   className,
   size = "sm",
+  version = false,
+  showEnvLabel = false,
 }: {
   size?: "sm" | "xl";
   className?: string;
+  version?: boolean;
+  showEnvLabel?: boolean;
 }) => (
-  <div className={cn("flex items-center", className)}>
-    <LangfuseIcon size={size === "sm" ? 16 : 20} />
-    <span
-      className={cn(
-        "font-mono font-semibold",
-        size === "sm" ? "ml-2 text-sm" : "ml-3 text-xl",
+  <div className={cn("flex gap-4 xl:flex-col xl:items-start", className)}>
+    {/* Environment Labeling for Langfuse Maintainers */}
+    {showEnvLabel && env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && (
+      <div
+        className={cn(
+          "flex items-center gap-2 self-stretch rounded-md  px-3 py-1  ring-1  xl:-mx-2",
+          env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "STAGING"
+            ? "bg-blue-100 text-blue-500 ring-blue-500"
+            : "bg-red-100 text-red-500 ring-red-500",
+        )}
+      >
+        <AlertTriangle size={16} />
+        <span className="whitespace-nowrap">
+          {["EU", "US"].includes(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION)
+            ? `PRODUCTION-${env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}`
+            : env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
+        </span>
+      </div>
+    )}
+    {/* Langfuse Logo */}
+    <div className="flex items-center">
+      <LangfuseIcon size={size === "sm" ? 16 : 20} />
+      <span
+        className={cn(
+          "font-mono font-semibold",
+          size === "sm" ? "ml-2 text-sm" : "ml-3 text-xl",
+        )}
+      >
+        Langfuse
+      </span>
+      {version && (
+        <a
+          href="https://github.com/langfuse/langfuse/releases"
+          target="_blank"
+          rel="noopener"
+          title="View releases on GitHub"
+          className="ml-2 text-xs text-gray-400"
+        >
+          {VERSION}
+        </a>
       )}
-    >
-      Langfuse
-    </span>
+    </div>
   </div>
 );

src/components/bookmark-trace.tsx

@@ -1,60 +0,0 @@
-import { StarIcon } from "lucide-react";
-
-import { Button } from "@/src/components/ui/button";
-import { api } from "@/src/utils/api";
-import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
-import { useEffect, useState } from "react";
-
-export function BookmarkTrace({
-  traceId,
-  projectId,
-  isBookmarked,
-  size = "sm",
-}: {
-  traceId: string;
-  projectId: string;
-  isBookmarked: boolean;
-  size?: "sm" | "xs";
-}) {
-  const utils = api.useUtils();
-  const [cacheIsBookmarked, setCacheIsBookmarked] = useState<boolean | null>(
-    null,
-  );
-  const localIsBookmarked = cacheIsBookmarked ?? isBookmarked;
-  const hasAccess = useHasAccess({ projectId, scope: "traces:bookmark" });
-  const mutBookmarkTrace = api.traces.bookmark.useMutation({
-    onSuccess: () => {
-      setCacheIsBookmarked(!localIsBookmarked);
-      void utils.traces.invalidate();
-    },
-  });
-
-  useEffect(() => {
-    setCacheIsBookmarked(null);
-  }, [isBookmarked]);
-
-  const handleBookmarkClick = () => {
-    if (!hasAccess) return;
-    void mutBookmarkTrace.mutateAsync({
-      traceId,
-      projectId,
-      bookmarked: !localIsBookmarked,
-    });
-  };
-
-  return (
-    <Button
-      variant="ghost"
-      size={size}
-      onClick={handleBookmarkClick}
-      disabled={!hasAccess}
-      loading={mutBookmarkTrace.isLoading}
-    >
-      <StarIcon
-        className={`h-4 w-4 ${
-          localIsBookmarked ? "fill-current text-yellow-500" : "text-gray-500"
-        }`}
-      />
-    </Button>
-  );
-}

src/components/date-picker.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import * as React from "react";
-import { Calendar as CalendarIcon } from "lucide-react";
+import { Calendar as CalendarIcon, X } from "lucide-react";
 import { Button } from "@/src/components/ui/button";
 import { Calendar } from "@/src/components/ui/calendar";
 import {
@@ -39,36 +39,50 @@ export type AvailableDateRangeSelections =
 export function DatePicker({
   date,
   onChange,
+  clearable = false,
   className,
 }: {
   date?: Date | undefined;
   onChange: (date: Date | undefined) => void;
+  clearable?: boolean;
   className?: string;
 }) {
   return (
-    <Popover>
-      <PopoverTrigger asChild>
+    <div className="flex flex-row gap-2 align-middle">
+      <Popover>
+        <PopoverTrigger asChild>
+          <Button
+            variant={"outline"}
+            className={cn(
+              "justify-start text-left font-normal",
+              !date && "text-muted-foreground",
+              className,
+            )}
+          >
+            <CalendarIcon className="mr-2 h-4 w-4" />
+            {date ? format(date, "PPP") : <span>Pick a date</span>}
+          </Button>
+        </PopoverTrigger>
+        <PopoverContent className="w-auto p-0">
+          <Calendar
+            mode="single"
+            selected={date}
+            onSelect={(d) => onChange(d)}
+            initialFocus
+          />
+        </PopoverContent>
+      </Popover>
+      {date && clearable && (
         <Button
-          variant={"outline"}
-          className={cn(
-            "justify-start text-left font-normal",
-            !date && "text-muted-foreground",
-            className,
-          )}
+          variant="ghost"
+          size="icon"
+          onClick={() => onChange(undefined)}
+          title="reset date"
         >
-          <CalendarIcon className="mr-2 h-4 w-4" />
-          {date ? format(date, "PPP") : <span>Pick a date</span>}
+          <X size={14} />
         </Button>
-      </PopoverTrigger>
-      <PopoverContent className="w-auto p-0">
-        <Calendar
-          mode="single"
-          selected={date}
-          onSelect={(d) => onChange(d)}
-          initialFocus
-        />
-      </PopoverContent>
-    </Popover>
+      )}
+    </div>
   );
 }
 
@@ -191,7 +205,7 @@ export function DatePickerWithRange({
             {DEFAULT_DATE_RANGE_SELECTION}
           </SelectItem>
           {dateTimeAggregationOptions.toReversed().map((item) => (
-            <SelectItem key={item} value={`${item}`}>
+            <SelectItem key={item} value={item}>
               {item}
             </SelectItem>
           ))}

src/components/delete-trace.tsx

@@ -1,6 +1,6 @@
 import { TrashIcon } from "lucide-react";
 import { useRouter } from "next/router";
-
+import { useState } from "react";
 import { Button } from "@/src/components/ui/button";
 import { api } from "@/src/utils/api";
 import {
@@ -9,6 +9,7 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { usePostHog } from "posthog-js/react";
 
 export function DeleteTrace({
   traceId,
@@ -19,13 +20,16 @@ export function DeleteTrace({
   projectId: string;
   isTableAction?: boolean;
 }) {
+  const [isDeleted, setIsDeleted] = useState(false);
   const router = useRouter();
   const utils = api.useUtils();
+  const posthog = usePostHog();
 
   const hasAccess = useHasAccess({ projectId, scope: "traces:delete" });
 
-  const mutDeleteTrace = api.traces.delete.useMutation({
+  const mutDeleteTraces = api.traces.deleteMany.useMutation({
     onSuccess: () => {
+      setIsDeleted(true);
       if (!isTableAction) {
         void router
           .push(`/project/${projectId}/traces`)
@@ -41,14 +45,14 @@ export function DeleteTrace({
   }
 
   return (
-    <Popover>
+    <Popover key={traceId}>
       <PopoverTrigger asChild>
         {isTableAction ? (
           <Button variant="ghost" size="xs">
             <TrashIcon className="h-4 w-4" />
           </Button>
         ) : (
-          <Button variant="outline" type="button">
+          <Button variant="outline" type="button" size="icon">
             <TrashIcon className="h-4 w-4" />
           </Button>
         )}
@@ -63,10 +67,16 @@ export function DeleteTrace({
           <Button
             type="button"
             variant="destructive"
-            loading={mutDeleteTrace.isLoading}
-            onClick={() =>
-              void mutDeleteTrace.mutateAsync({ traceId, projectId })
-            }
+            loading={mutDeleteTraces.isLoading || isDeleted}
+            onClick={() => {
+              void mutDeleteTraces.mutateAsync({
+                traceIds: [traceId],
+                projectId,
+              });
+              posthog.capture("trace:delete", {
+                source: isTableAction ? "table-single-row" : "trace",
+              });
+            }}
           >
             Delete trace
           </Button>