chore: release v2.6.0

Co-authored-by: RichardKruemmel <richard.kruemmel@code.berlin>

.env.dev.example

@@ -16,8 +16,21 @@ NEXTAUTH_SECRET="secret"
 
 # Langfuse experimental features
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
 SALT="salt"
 
 # Email
 EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# S3 storage
+S3_ENDPOINT=
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
+S3_BUCKET_NAME=
+S3_REGION=
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1

.env.local.example

@@ -1,21 +1,14 @@
 # When adding additional environment variables, the schema in "/src/env.mjs"
 # should be updated accordingly.
+
 # Prisma
 # https://www.prisma.io/docs/reference/database-reference/connection-urls#env
 DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
 DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
 # Next Auth
-# You can generate a new secret on the command line with:
-# openssl rand -base64 32
-# https://next-auth.js.org/configuration/options#secret
 NEXTAUTH_SECRET="secret"
 NEXTAUTH_URL="http://localhost:3000"
 
 # feature flag to enable experimental features locally
 LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
 SALT="salt"
-
-
-# Email
-EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
-SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.

.env.prod.example

@@ -1,32 +1,75 @@
-# Assuming deployment on Vercel with Postgres database on Supabase
-NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
-NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"
+# More information: https://langfuse.com/docs/deployment/self-host
+
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
 
 # Prisma
-
 # https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:[pw]@db.[db_id].supabase.co:5432/postgres"
-DATABASE_URL="postgres://postgres:[pw]@db.[db_id].supabase.co:6543/postgres?pgbouncer=true&connection_limit=1"
+# DATABASE_URL supports pooled connections, but then you need to set DIRECT_URL
+DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
+# DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
 # SHADOW_DATABASE_URL=
 
 # Next Auth
-
 # NEXTAUTH_URL does not need to be set when deploying on Vercel
-# NEXTAUTH_URL="http://localhost:3000"
-
-# AUTH_REDIRECT_PROXY_URL used to proxy oauth callbacks on e.g. preview deployments. optional.
-# AUTH_REDIRECT_PROXY_URL="https://example.com/api/auth"
+NEXTAUTH_URL="http://localhost:3000"
 
 # You can generate a new secret on the command line with:
 # openssl rand -base64 32
 # https://next-auth.js.org/configuration/options#secret
 NEXTAUTH_SECRET="secret"
 SALT="salt"
-# Sentry; set via Vercel integration
-# NEXT_PUBLIC_SENTRY_DSN=
-# NEXT_SENTRY_ORG=
-# NEXT_SENTRY_PROJECT=
-# SENTRY_AUTH_TOKEN=
+
+# Docker only, optional
+# PORT=3000
+# HOSTNAME=localhost
+
+# Default project, optional
+# LANGFUSE_DEFAULT_PROJECT_ID=
+# LANGFUSE_DEFAULT_PROJECT_ROLE=
+
+# Enable experimental features, optional
+# NEXT_PUBLIC_ENABLE_EXPERIMENTAL_FEATURES
+
+# Auth, optional configuration
+# AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
+# AUTH_DISABLE_USERNAME_PASSWORD=true
+
+# SSO, each group is optional
+# AUTH_GOOGLE_CLIENT_ID=
+# AUTH_GOOGLE_CLIENT_SECRET=
+# AUTH_GITHUB_CLIENT_ID=
+# AUTH_GITHUB_CLIENT_SECRET=
+# AUTH_AZURE_AD_CLIENT_ID=
+# AUTH_AZURE_AD_CLIENT_SECRET=
+# AUTH_AZURE_AD_TENANT_ID=
+
+# Transactional email, optional
+# Defines the email address to use as the from address.
+# EMAIL_FROM_ADDRESS=
+# Defines the connection url for smtp server. 
+# SMTP_CONNECTION_URL=
+
+# S3 storage, optional, used for exports from the UI
+# S3_ENDPOINT=
+# S3_ACCESS_KEY_ID=
+# S3_SECRET_ACCESS_KEY=
+# S3_BUCKET_NAME=
+# S3_REGION=
+
+# Exports are streamed to S3 in pages to avoid memory issues
+# The page size can be adjusted if needed to optimize performance
+# DB_EXPORT_PAGE_SIZE=1000
+
+
+
+
+### START Langfuse Cloud Config
+# Used for Langfuse Cloud deployments
+# Not recommended for self-hosted deployments as these are NOT COVERED BY SEMVER
+
+# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="US"
+# NEXTAUTH_COOKIE_DOMAIN=".langfuse.com"
 
 # LANGFUSE_TEAM_SLACK_WEBHOOK=
 # LANGFUSE_NEW_USER_SIGNUP_WEBHOOK=
@@ -35,24 +78,19 @@ SALT="salt"
 # NEXT_PUBLIC_POSTHOG_HOST=
 # NEXT_PUBLIC_POSTHOG_KEY=
 
-# Id of demo project to automatically assign new users to
+# Sentry
+# NEXT_PUBLIC_SENTRY_DSN=
+# NEXT_SENTRY_ORG=
+# NEXT_SENTRY_PROJECT=
+# SENTRY_AUTH_TOKEN=
+
+# Betterstack
+# LANGFUSE_TEAM_BETTERSTACK_TOKEN=
+
+# Demo project that users can use to try the platform
 # NEXT_PUBLIC_DEMO_PROJECT_ID=
 
-# Auth, each group is optional
-AUTH_GOOGLE_CLIENT_ID=
-AUTH_GOOGLE_CLIENT_SECRET=
-AUTH_GITHUB_CLIENT_ID=
-AUTH_GITHUB_CLIENT_SECRET=
-# AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
-# AUTH_DISABLE_USERNAME_PASSWORD=true
+# Crisp chat
+# NEXT_PUBLIC_CRISP_WEBSITE_ID=
 
-# Email
-EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
-SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
-
-# S3 storage (eg for exports), necessary for serverless deployments
-S3_ENDPOINT=""
-S3_ACCESS_KEY_ID=""
-S3_SECRET_ACCESS_KEY=""
-S3_BUCKET_NAME=""
-S3_REGION=""
+### END Langfuse Cloud Config

.github/DISCUSSION_TEMPLATE/ideas.yml

@@ -0,0 +1,11 @@
+body:
+  - type: textarea
+    attributes:
+      label: Describe the feature or potential improvement
+      description: Please describe the change as clear and concise as possible. Remember to add context as to why you believe this is needed.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the change here. If your idea is related to any issues or discussions, link them here.

.github/workflows/pipeline.yml

@@ -33,10 +33,6 @@ jobs:
   test-docker-build:
     runs-on: ubuntu-latest
     env:
-      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
-      NEXTAUTH_SECRET: "secret"
-      SALT: "salt"
-      NEXTAUTH_URL: "http://localhost:3030"
       REGISTRY: ghcr.io
       IMAGE_NAME: ${{ github.repository }}
 
@@ -54,11 +50,6 @@ jobs:
         with:
           context: .
           push: false
-          build-args: |
-            DATABASE_URL=${{ env.DATABASE_URL }}
-            NEXTAUTH_SECRET=${{ env.NEXTAUTH_SECRET }}
-            NEXTAUTH_URL=${{ env.NEXTAUTH_URL }}
-            SALT=${{ env.SALT }}
 
   tests:
     runs-on: ubuntu-latest
@@ -152,10 +143,6 @@ jobs:
     environment: "protected branches"
     runs-on: ubuntu-latest
     env:
-      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
-      NEXTAUTH_SECRET: "secret"
-      SALT: "salt"
-      NEXTAUTH_URL: "http://localhost:3030"
       REGISTRY: ghcr.io
       IMAGE_NAME: ${{ github.repository }}
     permissions:
@@ -198,8 +185,3 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            DATABASE_URL=${{ env.DATABASE_URL }}
-            NEXTAUTH_SECRET=${{ env.NEXTAUTH_SECRET }}
-            NEXTAUTH_URL=${{ env.NEXTAUTH_URL }}
-            SALT=${{ env.SALT }}

.gitignore

@@ -48,4 +48,8 @@ yarn-error.log*
 /generated/typescript-server
 
 # openapi spec that is copied during build
-/public/openapi*.yml
+/public/openapi*.yml
+
+
+# vscode
+.devcontainer

CONTRIBUTING.md

@@ -2,22 +2,30 @@
 
 First off, thanks for taking the time to contribute! ❤️
 
-Langfuse is an open-source observability and analytics solution for LLM-based applications. We welcome contributions through GitHub pull requests. This document outlines our conventions regarding development workflow, commit message formatting, contact points, and other resources. Our goal is to simplify the process and ensure that your contributions are easily accepted.
+The best ways to contribute to Langfuse:
 
-We gratefully welcome improvements to documentation as well as to code.
+- Submit and vote on [Ideas](https://github.com/orgs/langfuse/discussions/categories/ideas)
+- Create and comment on [Issues](https://github.com/langfuse/langfuse/issues)
+- Open a PR.
+
+We welcome contributions through GitHub pull requests. This document outlines our conventions regarding development workflow, commit message formatting, contact points, and other resources. Our goal is to simplify the process and ensure that your contributions are easily accepted.
+
+We gratefully welcome improvements to documentation ([docs repo](https://github.com/langfuse/langfuse-docs)), the core application (this repo) and the SDKs ([Python](https://github.com/langfuse/langfuse-python), [JS](https://github.com/langfuse/langfuse-js)).
 
 The maintainers are available on [Discord](https://langfuse.com/discord) in case you have any questions.
 
-> And if you like the project, but just don't have time to contribute, that's fine. There are other easy ways to support the project and show your appreciation, which we would also be very happy about:
+> And if you like the project, but just don't have time to contribute code, that's fine. There are other easy ways to support the project and show your appreciation, which we would also be very happy about:
 >
 > - Star the project;
 > - Tweet about it;
 > - Refer to this project in your project's readme;
+> - Submit and vote on [Ideas](https://github.com/orgs/langfuse/discussions/categories/ideas);
+> - Create and comment on [Issues](https://github.com/langfuse/langfuse/issues);
 > - Mention the project at local meetups and tell your friends/colleagues.
 
 ## Making a change
 
-_Before making any significant changes, please [open an issue](https://github.com/langfuse/langfuse/issues)._ Discussing your proposed changes ahead of time will make the contribution process smooth for everyone.
+_Before making any significant changes, please [open an issue](https://github.com/langfuse/langfuse/issues)._ Discussing your proposed changes ahead of time will make the contribution process smooth for everyone. Large changes that were not discussed in an issue may be rejected.
 
 Once we've discussed your changes and you've got your code ready, make sure that tests are passing and open your pull request.
 

Dockerfile

@@ -1,19 +1,11 @@
 # Base image
 FROM node:20-alpine AS base
-ARG DATABASE_URL
-ARG NEXTAUTH_SECRET
-ARG NEXTAUTH_URL
-ARG SALT
 
 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat
 
 FROM base AS deps
-ARG DATABASE_URL
-ARG NEXTAUTH_SECRET
-ARG NEXTAUTH_URL
-ARG SALT
 
 WORKDIR /app
 
@@ -29,10 +21,6 @@ RUN \
 
 # Rebuild the source code only when needed
 FROM base AS builder
-ARG DATABASE_URL
-ARG NEXTAUTH_SECRET
-ARG NEXTAUTH_URL
-ARG SALT
 
 WORKDIR /app
 COPY --from=deps /app/node_modules ./node_modules
@@ -46,6 +34,9 @@ RUN rm -f ./src/middleware.ts
 # Uncomment the following line in case you want to disable telemetry during the build.
 ENV NEXT_TELEMETRY_DISABLED 1
 
+# Disable validation of environment variables during build
+ENV DOCKER_BUILD 1
+
 # Generate prisma client
 RUN npx prisma generate
 
@@ -54,10 +45,6 @@ RUN npm run build
 
 # Production image, copy all the files and run next
 FROM base AS runner
-ARG DATABASE_URL
-ARG NEXTAUTH_SECRET
-ARG NEXTAUTH_URL
-ARG SALT
 
 RUN apk add --no-cache dumb-init
 
@@ -66,6 +53,8 @@ WORKDIR /app
 ENV NODE_ENV production
 # Uncomment the following line in case you want to disable telemetry during runtime.
 ENV NEXT_TELEMETRY_DISABLED 1
+# Needed to re-enable validation of environment variables during runtime
+ENV DOCKER_BUILD 0
 
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs

README.md

@@ -2,42 +2,43 @@
    <a href="https://langfuse.com">
       <h1>🪢 Langfuse</h1>
    </a>
+ <div>
+      <h3> <a href="https://cloud.langfuse.com">
+         <strong>Sign up</strong>
+       </a> · 
+      <a href="https://langfuse.com/docs/deployment/self-host">
+         <strong>Self Host Langfuse</strong>
+      </a> · 
+      <a href="https://langfuse.com/demo">
+         <strong>Demo Project (live data)</strong>
+      </a>
+      </h3>
    <h3>
-      Open source observability & analytics for LLM-based applications
+      Langfuse is the open source LLM engineering platform.   
    </h3>
-   <div>
-      <strong>Observability:</strong> Explore and debug complex logs & traces in a visual UI
-   </div>
-   <div>
-      <strong>Analytics:</strong> Measure & improve costs, latency and response quality
+    <div>
+     Debug, analyze and iterate - together
    </div>
    </br>
    <div>
-      <a href="https://discord.gg/7NXusRtqYU">
-         <strong>Join the Langfuse Discord »</strong>
-      </a>
-      </br>
-      <a href="https://langfuse.com">
-         <strong>langfuse.com</strong>
-      </a> ·
       <a href="https://langfuse.com/docs">
          <strong>Docs</strong>
       </a> ·
-      <a href="https://github.com/langfuse/langfuse/issues/new?labels=%F0%9F%90%9E%E2%9D%94+unconfirmed+bug&projects=&template=bug_report.yml&title=bug%3A+">
+      <a href="https://langfuse.com/issue">
          <strong>Report Bug</strong>
       </a> ·
-      <a href="https://github.com/langfuse/langfuse/issues/new?assignees=&labels=%E2%9C%A8+enhancement&projects=&template=feature_request.yml&title=feat%3A+">
+      <a href="https://langfuse.com/idea">
          <strong>Feature Request</strong>
-      </a>
+      </a> ·
+      <a href="https://langfuse.com/changelog">
+         <strong>Changelog</strong>
+      </a> ·
+      <a href="https://langfuse.com/discord">
+         <strong>Discord</strong>
+      </a> 
    </div>
    </br>
-   <div>
       <img src="https://img.shields.io/badge/License-MIT-red.svg?style=flat-square" alt="MIT License">
-      <a href="https://discord.gg/7NXusRtqYU"><img src="https://img.shields.io/discord/1111061815649124414?style=flat-square&logo=Discord&logoColor=white&label=Discord&color=%23434EE4" alt="Discord"></a>
-      <a href="https://github.com/langfuse/langfuse"><img src="https://img.shields.io/github/stars/langfuse/langfuse?style=flat-square&logo=GitHub&label=langfuse%2Flangfuse" alt="Github Repo Stars"></a>
-      <a href="https://github.com/langfuse/langfuse/releases"><img src="https://img.shields.io/github/v/release/langfuse/langfuse?include_prereleases&style=flat-square" alt="langfuse releases"></a>
-      <a href="https://github.com/langfuse/langfuse/actions/workflows/pipeline.yml?query=branch:main"><img src="https://img.shields.io/github/actions/workflow/status/langfuse/langfuse/pipeline.yml?style=flat-square&label=All%20tests" alt="CI test status"></a>
-      <a href="https://status.langfuse.com"><img src="https://uptime.betterstack.com/status-badges/v1/monitor/udlc.svg" alt="Uptime Status"/></a>
       <a href="https://www.ycombinator.com/companies/langfuse"><img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23"></a>
       <a href="https://github.com/langfuse/langfuse/pkgs/container/langfuse"><img alt="Docker Image" src="https://img.shields.io/badge/docker-langfuse-blue?logo=Docker&logoColor=white&style=flat-square"></a>
       <a href="https://www.npmjs.com/package/langfuse"><img src="https://img.shields.io/npm/v/langfuse?style=flat-square&label=npm+langfuse" alt="langfuse npm package"></a>
@@ -45,61 +46,44 @@
    </div>
 </div>
 </br>
-</div>
-</br>
 
-## What is Langfuse?
+## Overview
 
-Langfuse is an open source observability & analytics solution for LLM-based applications. It is mostly geared towards production usage but some users also use it for local development of their LLM applications.
+### Develop
 
-Langfuse is focused on applications built on top of LLMs. Many new abstractions and common best practices evolved recently, e.g. agents, chained prompts, embedding-based retrieval, LLM access to REPLs & APIs. These make applications more powerful but also unpredictable for developers as they cannot fully anticipate how changes impact the quality, cost and overall latency of their application. Thus Langfuse helps to monitor and debug these applications.
+- **Observability:** Instrument your app and start ingesting traces to Langfuse ([Quickstart](https://langfuse.com/docs/get-started), [Integrations](https://langfuse.com/docs/integrations) [Tracing](https://langfuse.com/docs/tracing))
+- **Langfuse UI:** Inspect and debug complex logs ([Demo](https://langfuse.com/docs/demo), [Tracing](https://langfuse.com/docs/tracing))
+- **Prompts:** Manage, version and deploy prompts from within Langfuse ([Prompt Management](https://langfuse.com/docs/prompts))
 
-**Demo (2 min)**
+### Monitor
+
+- **Analytics:** Track metrics (cost, latency, quality) and gain insights from dashboards & data exports ([Analytics](https://langfuse.com/docs/analytics))
+- **Evals:** Collect and calculate scores for your LLM completions ([Scores & Evaluations](https://langfuse.com/docs/scores))
+  - Run model-based evaluations ([Model-based evaluations](https://langfuse.com/docs/scores/model-based-evals))
+  - Collect user feedback ([User Feedback](https://langfuse.com/docs/scores/user-feedback))
+  - Manually score observations in Langfuse ([Manual Scores](https://langfuse.com/docs/scores/manually))
+
+### Test
+
+- **Experiments:** Track and test app behaviour before deploying a new version
+  - Datasets let you test expected in and output pairs and benchmark performance before deployiong ([Datasets](https://langfuse.com/docs/datasets))
+  - Track versions and releases in your application ([Experimentation](https://langfuse.com/docs/experimentation), [Prompt Management](https://langfuse.com/docs/prompts))
+
+### Video: Langfuse in two minutes
 
 https://github.com/langfuse/langfuse/assets/2834609/6041347a-b517-4a11-8737-93ef8f8af49f
 
 _Muted by default, enable sound for voice-over_
 
-Explore demo project in Langfuse here (free account required): https://langfuse.com/demo
-
-### Observability
-
-Langfuse offers an admin UI to explore the ingested data.
-
-- Nested view of LLM app executions; detailed information along the traces on: latency, cost, scores
-- Segment execution traces by user feedback, to e.g. identify production issues
-
-### Analytics
-
-Reporting on
-
-- Token usage by model
-- Volume of traces
-- Scores/evals
-
-Broken down by
-
-- Users
-- Releases
-- Prompt/chain versions
-- Prompt/chain types
-- Time
-
-→ Expect releases with more ways to analyze the data over the next weeks.
-
 ## Get started
 
-### Step 1: Run Server
+### Langfuse Cloud
 
-#### Langfuse Cloud
+Managed deployment by the Langfuse team, generous free-tier (hobby plan), no credit card required.
 
-Managed deployment by the Langfuse team, generous free-tier (hobby plan) available, no credit card required.
+**[» Langfuse Cloud](https://cloud.langfuse.com)**
 
-Links: [Create account](https://cloud.langfuse.com), [learn more](https://cloud.langfuse.com)
-
-#### Localhost
-
-Requirements: docker, docker compose (e.g. using Docker Desktop)
+### Localhost (docker)
 
 ```bash
 # Clone repository
@@ -110,143 +94,84 @@ cd langfuse
 docker compose up -d
 ```
 
-#### Self-host (Docker)
+[→ Learn more about deploying locally](https://langfuse.com/docs/deployment/local)
 
-[→ Instructions](https://langfuse.com/docs/deployment/self-host)
+### Self-host (docker)
 
-[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/gmbqa_)
+Langfuse is simple to self-host and keep updated. It currently requires only a single docker container.
+[→ Self Hosting Instructions](https://langfuse.com/docs/deployment/self-host)
 
-### Step 2: Data ingestion
+Templated deployments: [Railway, GCP Cloud Run, AWS Fargate, Kubernetes and others](https://langfuse.com/docs/deployment/self-host#platform-specific-information)
 
-#### SDKs to instrument application
+## Get Started
 
-Fully async, typed SDKs to instrument any LLM application. Currently available for Python & JS/TS.
+### API Keys
 
-→ [Guide](https://langfuse.com/docs/guides/sdk-integration) with an example of how the SDK can be used
+You require a Langfuse public and secret key to get started. Sign up [here](https://cloud.langfuse.com) and find them in your project settings.
 
-| Package                                                                                                                                             | Description                      | Links                                                                                                          |
-| --------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------- | -------------------------------------------------------------------------------------------------------------- |
-| [![PyPI Version](https://img.shields.io/pypi/v/langfuse.svg?style=flat-square&label=pypi+langfuse)](https://pypi.python.org/pypi/langfuse)          | Python                           | [docs](https://langfuse.com/docs/integrations/sdk/python), [repo](https://github.com/langfuse/langfuse-python) |
-| [![npm Version](https://img.shields.io/npm/v/langfuse?style=flat-square&label=npm+langfuse)](https://www.npmjs.com/package/langfuse)                | JS/TS: Node >= 18, Edge runtimes | [docs](https://langfuse.com/docs/integrations/sdk/typescript), [repo](https://github.com/langfuse/langfuse-js) |
-| [![npm package](https://img.shields.io/npm/v/langfuse-node?style=flat-square&label=npm+langfuse-node)](https://www.npmjs.com/package/langfuse-node) | JS/TS: Node <18                  | [docs](https://langfuse.com/docs/integrations/sdk/typescript), [repo](https://github.com/langfuse/langfuse-js) |
+### Ingesting Data · Instrumenting Your Application
 
-#### Langchain applications
+Note: We recommend using our fully async, typed [SDKs](https://langfuse.com/docs/sdk) that allow you to instrument any LLM application with any underlying model. They are available in [Python](https://langfuse.com/docs/sdk/python) & [JS/TS](https://langfuse.com/docs/sdk/typescript). The SDKs will always be the most fully featured and stable way to ingest data into Langfuse.
 
-The Langfuse callback handler automatically instruments Langchain applications. Currently available for Python and JS/TS.
+You may want to use another integration to get started quickly or implement a use case that we do not yet support. However, we recommend to migrate to the Langfuse SDKs over time to ensure performance and stability.
 
-**Python**
+See our the [→ Quickstart](https://langfuse.com/docs/get-started) to get started in integrating Langfuse.
 
-```shell
-pip install langfuse
-```
+### Integrations
 
-```python
-# Initialize Langfuse handler
-from langfuse.callback import CallbackHandler
-handler = CallbackHandler(PUBLIC_KEY, SECRET_KEY)
+| Integration                                              | Supports      | Description                                                                     |
+| -------------------------------------------------------- | ------------- | ------------------------------------------------------------------------------- |
+| [**SDK** - _recommended_](https://langfuse.com/docs/sdk) | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility.                     |
+| [OpenAI](https://langfuse.com/docs/openai)               | Python        | Automated instrumentation using drop-in replacement of OpenAI SDK.              |
+| [Langchain](https://langfuse.com/docs/langchain)         | Python, JS/TS | Automated instrumentation by passing callback handler to Langchain application. |
+| [API](https://langfuse.com/docs/api)                     |               | Directly call the public API. OpenAPI spec available.                           |
 
-# Setup Langchain
-from langchain.chains import LLMChain
-...
-chain = LLMChain(llm=llm, prompt=prompt)
+External projects/packages that integrate with Langfuse:
 
-# Add Langfuse handler as callback
-chain.run(input="<user_input", callbacks=[handler])
-```
+| Name                                           | Description                                                                                                                                      |
+| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [LiteLLM](/https://langfuse.comdocs/litellm)   | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
+| [Flowise](https://langfuse.com/docs/flowise)   | JS/TS no-code builder for customized LLM flows.                                                                                                  |
+| [Langflow](https://langfuse.com/docs/langflow) | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows.                          |
 
-→ [Langchain integration docs for Python](https://langfuse.com/docs/integrations/langchain/python)
+## Questions and feedback
 
-**JS/TS**
+### Ideas and roadmap
 
-→ [Langchain integration docs for JS/TS](https://langfuse.com/docs/integrations/langchain/typescript)
+- [GitHub Discussions](https://github.com/orgs/langfuse/discussions)
+- [Feature Requests](https://langfuse.com/idea)
 
-#### Add scores/evaluations to traces (optional)
+### Support and feedback
 
-Quality/evaluation of traces is tracked via scores ([docs](https://langfuse.com/docs/scores)). Scores are related to traces and optionally to observations. Scores can be added via:
+In order of preference the best way to communicate with us:
 
-- **Backend SDKs** (see docs above): `{trace, event, span, generation}.score()`
-- **API** (see docs below): `POST /api/public/scores`
-- **Client-side using Web SDK**, e.g. to capture user feedback or other user-based quality metrics:
-
-  ```sh
-  npm install langfuse
-  ```
-
-  ```ts
-  // Client-side (browser)
-
-  import { LangfuseWeb } from "langfuse";
-
-  const langfuseWeb = new LangfuseWeb({
-    publicKey: process.env.LANGFUSE_PUBLIC_KEY,
-  });
-
-  // frontend handler (example: React)
-  export function UserFeedbackComponent(props: { traceId: string }) {
-    const handleUserFeedback = async (value: number) => {
-      await langfuseWeb.score({
-        traceId: props.traceId,
-        name: "user_feedback",
-        value,
-      });
-    };
-    return (
-      <div>
-        <button onClick={() => handleUserFeedback(1)}>👍</button>
-        <button onClick={() => handleUserFeedback(-1)}>👎</button>
-      </div>
-    );
-  }
-  ```
-
-#### API
-
-[**Api reference**](https://langfuse.com/docs/integrations/api)
-
-- POST/PATCH routes to ingest data
-- GET routes to use data in downstream applications (e.g. embedded analytics)
-
-## Questions / Feedback
-
-The maintainers are very active in the Langfuse [Discord](https://langfuse.com/discord) and are happy to answer questions or discuss feedback/ideas regarding the future of the project.
+- [GitHub Discussions](https://github.com/orgs/langfuse/discussions): Contribute [ideas](https://langfuse.com/idea) [support requests](https://github.com/orgs/langfuse/discussions/categories/support) and [report bugs](https://github.com/langfuse/langfuse/issues/new?labels=%F0%9F%90%9E%E2%9D%94+unconfirmed+bug&projects=&template=bug_report.yml&title=bug%3A+) (preferred as we create a permanent, indexed artifact for other community members)
+- [Discord](https://langfuse.com/discord): For community support and to chat directly with maintainers
+- Privately: Email contact at langfuse dot com
 
 ## Contributing to Langfuse
 
-Join the community [on Discord](https://discord.gg/7NXusRtqYU).
-
-To contribute, send us a PR, raise a GitHub issue, or email at contributing@langfuse.com
-
-### Development setup
-
-See [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to setup a development environment.
+- Vote on [Ideas](https://github.com/orgs/langfuse/discussions/categories/ideas)
+- Raise and comment on [Issues](https://github.com/langfuse/langfuse/issues)
+- Open a PR - see [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to setup a development environment.
 
 ## License
 
-Langfuse is MIT licensed, except for `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.
+This repository is MIT licensed, except for the `ee/` folder. See [LICENSE](LICENSE) and [docs](https://langfuse.com/docs/open-source) for more details.
 
 ## Misc
 
-### Upgrade Langfuse (localhost)
+### GET API to export your data
 
-```bash
-# Stop server and db
-docker compose down
+[**GET routes**](https://langfuse.com/docs/integrations/api) to use data in downstream applications (e.g. embedded analytics).
 
-# Pull latest changes
-git pull
-docker-compose pull
+### Security & Privacy
 
-# Run server and db
-docker compose up -d
-```
-
-### Run Langfuse in CI for integration tests
-
-Checkout GitHub Actions workflows of [Python SDK](https://github.com/langfuse/langfuse-python/blob/main/.github/workflows/ci.yml) and [JS/TS SDK](https://github.com/langfuse/langfuse-js/blob/main/.github/workflows/ci.yml).
+We take data security and privacy seriously. Please refer to our [Security and Privacy](https://langfuse.com/security) page for more information.
 
 ### Telemetry
 
-By default, Langfuse automatically reports basic usage statistics to a centralized server (PostHog).
+By default, Langfuse automatically reports basic usage statistics of self-hosted instances to a centralized server (PostHog).
 
 This helps us to:
 

docker-compose.build.yml

@@ -4,17 +4,11 @@ services:
   langfuse-server:
     build:
       dockerfile: Dockerfile
-      args:
-        - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-        - NEXTAUTH_SECRET=mysecret
-        - SALT=mysalt
-        - NEXTAUTH_URL=http://localhost:3000
     depends_on:
       - db
     ports:
       - "3000:3000"
     environment:
-      - NODE_ENV=production
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt

docker-compose.yml

@@ -8,7 +8,6 @@ services:
     ports:
       - "3000:3000"
     environment:
-      - NODE_ENV=production
       - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
       - NEXTAUTH_SECRET=mysecret
       - SALT=mysalt

fern/apis/server/definition/commons.yml

@@ -18,16 +18,25 @@ types:
       public:
         type: optional<boolean>
         docs: Public traces are accessible via url without login
-  TraceWithDetails:
+  TraceWithDetails: # GET /traces
     extends: Trace
     properties:
+      htmlPath:
+        type: string
+        docs: Path of trace in Langfuse UI
+      latency:
+        type: double
+        docs: Latency of trace in seconds
+      totalCost:
+        type: double
+        docs: Cost of trace in USD.
       observations:
         type: list<string>
         docs: List of observation ids
       scores:
         type: list<string>
         docs: List of score ids
-  TraceWithFullDetails:
+  TraceWithFullDetails: # GET traces/[traceID]
     extends: Trace
     properties:
       observations: list<ObservationsView>
@@ -72,6 +81,7 @@ types:
       calculatedInputCost: optional<double>
       calculatedOutputCost: optional<double>
       calculatedTotalCost: optional<double>
+      latency: optional<double>
 
   Usage:
     properties:
@@ -132,6 +142,9 @@ types:
     enum:
       - CHARACTERS
       - TOKENS
+      - MILLISECONDS
+      - SECONDS
+      - IMAGES
   ObservationLevel:
     enum:
       - DEBUG

fern/apis/server/definition/ingestion.yml

@@ -128,6 +128,7 @@ types:
   TraceBody:
     properties:
       id: optional<string>
+      timestamp: optional<datetime>
       name: optional<string>
       userId: optional<string>
       input: optional<unknown>

fern/apis/server/definition/metrics.yml

@@ -0,0 +1,48 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  pagination: ./utils/pagination.yml
+  commons: ./commons.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    daily:
+      docs: Get daily metrics of the Langfuse project
+      method: GET
+      path: /metrics/daily
+      request:
+        name: GetDailyMetricsRequest
+        query-parameters:
+          page: optional<integer>
+          limit: optional<integer>
+          traceName:
+            type: optional<string>
+            docs: Optional filter by the name of the trace
+          userId:
+            type: optional<string>
+            docs: Optional filter by the userId associated with the trace
+          tags:
+            type: optional<string>
+            allow-multiple: true
+            docs: Optional filter for metrics where traces include all of these tags
+      response: DailyMetrics
+types:
+  DailyMetrics:
+    properties:
+      data:
+        type: list<DailyMetricsDetails>
+        docs: A list of daily metrics, only days with ingested data are included.
+      meta: pagination.MetaResponse
+  DailyMetricsDetails:
+    properties:
+      date: date
+      countTraces: integer
+      totalCost: double
+      usage: list<UsageByModel>
+  UsageByModel:
+    docs: Daily usage of a given model. Usage corresponds to the unit set for the specific model (e.g. tokens).
+    properties:
+      model: string
+      inputUsage: integer
+      outputUsage: integer
+      totalUsage: integer

fern/apis/server/definition/score.yml

@@ -24,6 +24,14 @@ service:
           userId: optional<string>
           name: optional<string>
       response: Scores
+    delete:
+      docs: Delete a score
+      method: DELETE
+      path: /scores/{scoreId}
+      path-parameters:
+        scoreId:
+          type: string
+          docs: The unique langfuse identifier of a score
 types:
   CreateScoreRequest:
     properties:

generated/openapi-server/openapi.yml

@@ -387,6 +387,81 @@ paths:
                     $ref: '#/components/schemas/IngestionEvent'
               required:
                 - batch
+  /api/public/metrics/daily:
+    get:
+      description: Get daily metrics of the Langfuse project
+      operationId: metrics_daily
+      tags:
+        - Metrics
+      parameters:
+        - name: page
+          in: query
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: limit
+          in: query
+          required: false
+          schema:
+            type: integer
+            nullable: true
+        - name: traceName
+          in: query
+          description: Optional filter by the name of the trace
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: userId
+          in: query
+          description: Optional filter by the userId associated with the trace
+          required: false
+          schema:
+            type: string
+            nullable: true
+        - name: tags
+          in: query
+          description: Optional filter for metrics where traces include all of these tags
+          required: false
+          schema:
+            type: array
+            items:
+              type: string
+              nullable: true
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DailyMetrics'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
   /api/public/observations/{observationId}:
     get:
       description: Get a specific observation
@@ -761,6 +836,48 @@ paths:
             application/json:
               schema: {}
       security: *ref_0
+  /api/public/scores/{scoreId}:
+    delete:
+      description: Delete a score
+      operationId: score_delete
+      tags:
+        - Score
+      parameters:
+        - name: scoreId
+          in: path
+          description: The unique langfuse identifier of a score
+          required: true
+          schema:
+            type: string
+      responses:
+        '204':
+          description: ''
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security: *ref_0
   /api/public/sessions/{sessionId}:
     get:
       description: Get a session
@@ -983,6 +1100,17 @@ components:
       title: TraceWithDetails
       type: object
       properties:
+        htmlPath:
+          type: string
+          description: Path of trace in Langfuse UI
+        latency:
+          type: number
+          format: double
+          description: Latency of trace in seconds
+        totalCost:
+          type: number
+          format: double
+          description: Cost of trace in USD.
         observations:
           type: array
           items:
@@ -994,6 +1122,9 @@ components:
             type: string
           description: List of score ids
       required:
+        - htmlPath
+        - latency
+        - totalCost
         - observations
         - scores
       allOf:
@@ -1134,6 +1265,10 @@ components:
           type: number
           format: double
           nullable: true
+        latency:
+          type: number
+          format: double
+          nullable: true
       allOf:
         - $ref: '#/components/schemas/Observation'
     Usage:
@@ -1311,6 +1446,9 @@ components:
       enum:
         - CHARACTERS
         - TOKENS
+        - MILLISECONDS
+        - SECONDS
+        - IMAGES
     ObservationLevel:
       title: ObservationLevel
       type: string
@@ -1717,6 +1855,10 @@ components:
         id:
           type: string
           nullable: true
+        timestamp:
+          type: string
+          format: date-time
+          nullable: true
         name:
           type: string
           nullable: true
@@ -1933,6 +2075,60 @@ components:
       required:
         - successes
         - errors
+    DailyMetrics:
+      title: DailyMetrics
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            $ref: '#/components/schemas/DailyMetricsDetails'
+          description: A list of daily metrics, only days with ingested data are included.
+        meta:
+          $ref: '#/components/schemas/utilsMetaResponse'
+      required:
+        - data
+        - meta
+    DailyMetricsDetails:
+      title: DailyMetricsDetails
+      type: object
+      properties:
+        date:
+          type: string
+        countTraces:
+          type: integer
+        totalCost:
+          type: number
+          format: double
+        usage:
+          type: array
+          items:
+            $ref: '#/components/schemas/UsageByModel'
+      required:
+        - date
+        - countTraces
+        - totalCost
+        - usage
+    UsageByModel:
+      title: UsageByModel
+      type: object
+      description: >-
+        Daily usage of a given model. Usage corresponds to the unit set for the
+        specific model (e.g. tokens).
+      properties:
+        model:
+          type: string
+        inputUsage:
+          type: integer
+        outputUsage:
+          type: integer
+        totalUsage:
+          type: integer
+      required:
+        - model
+        - inputUsage
+        - outputUsage
+        - totalUsage
     Observations:
       title: Observations
       type: object

generated/postman/collection.json

@@ -320,7 +320,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"body\": {\n                \"id\": \"example\",\n                \"name\": \"example\",\n                \"userId\": \"example\",\n                \"input\": \"UNKNOWN\",\n                \"output\": \"UNKNOWN\",\n                \"sessionId\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"metadata\": \"UNKNOWN\",\n                \"tags\": [\n                    \"example\"\n                ],\n                \"public\": true\n            },\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"metadata\": \"UNKNOWN\"\n        }\n    ]\n}",
+              "raw": "{\n    \"batch\": [\n        {\n            \"type\": \"trace-create\",\n            \"body\": {\n                \"id\": \"example\",\n                \"timestamp\": \"1994-11-05T13:15:30Z\",\n                \"name\": \"example\",\n                \"userId\": \"example\",\n                \"input\": \"UNKNOWN\",\n                \"output\": \"UNKNOWN\",\n                \"sessionId\": \"example\",\n                \"release\": \"example\",\n                \"version\": \"example\",\n                \"metadata\": \"UNKNOWN\",\n                \"tags\": [\n                    \"example\"\n                ],\n                \"public\": true\n            },\n            \"id\": \"example\",\n            \"timestamp\": \"example\",\n            \"metadata\": \"UNKNOWN\"\n        }\n    ]\n}",
               "options": {
                 "raw": {
                   "language": "json"
@@ -332,6 +332,65 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Metrics",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Daily",
+          "request": {
+            "description": "Get daily metrics of the Langfuse project",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/metrics/daily?page=&limit=&traceName=&userId=&tags=",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "metrics",
+                "daily"
+              ],
+              "query": [
+                {
+                  "key": "page",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "limit",
+                  "value": "",
+                  "description": null
+                },
+                {
+                  "key": "traceName",
+                  "value": "",
+                  "description": "Optional filter by the name of the trace"
+                },
+                {
+                  "key": "userId",
+                  "value": "",
+                  "description": "Optional filter by the userId associated with the trace"
+                },
+                {
+                  "key": "tags",
+                  "value": "",
+                  "description": "Optional filter for metrics where traces include all of these tags"
+                }
+              ],
+              "variable": []
+            },
+            "header": [],
+            "method": "GET",
+            "auth": null,
+            "body": null
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,
@@ -623,6 +682,38 @@
             "body": null
           },
           "response": []
+        },
+        {
+          "_type": "endpoint",
+          "name": "Delete",
+          "request": {
+            "description": "Delete a score",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/scores/:scoreId",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "scores",
+                ":scoreId"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "scoreId",
+                  "value": "",
+                  "description": "The unique langfuse identifier of a score"
+                }
+              ]
+            },
+            "header": [],
+            "method": "DELETE",
+            "auth": null,
+            "body": null
+          },
+          "response": []
         }
       ]
     },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse-core",
-  "version": "2.1.2",
+  "version": "2.6.0",
   "private": true,
   "scripts": {
     "prebuild": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
@@ -16,6 +16,8 @@
     "test": "jest --runInBand",
     "test:watch": "jest --watch --runInBand",
     "test:e2e": "playwright test",
+    "infra:dev:up": "docker-compose  -f docker-compose.dev.yml up -d",
+    "infra:dev:down": "docker-compose -f docker-compose.dev.yml down",
     "db:migrate": "DISABLE_ERD=false npx prisma migrate dev",
     "db:reset": "npx prisma migrate reset",
     "db:seed": "npx prisma db seed",
@@ -28,13 +30,14 @@
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
-    "@aws-sdk/client-s3": "^3.501.0",
-    "@aws-sdk/s3-request-presigner": "^3.501.0",
+    "@aws-sdk/client-s3": "^3.507.0",
+    "@aws-sdk/lib-storage": "^3.511.0",
+    "@aws-sdk/s3-request-presigner": "^3.507.0",
     "@headlessui/react": "^1.7.18",
     "@heroicons/react": "^2.1.1",
     "@hookform/resolvers": "^3.3.4",
     "@next-auth/prisma-adapter": "^1.0.7",
-    "@prisma/client": "^5.8.1",
+    "@prisma/client": "^5.9.1",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.5",
     "@radix-ui/react-avatar": "^1.0.4",
@@ -57,44 +60,44 @@
     "@radix-ui/react-tooltip": "^1.0.7",
     "@react-email/components": "^0.0.14",
     "@react-email/render": "^0.0.12",
-    "@sentry/nextjs": "^7.99.0",
-    "@sentry/profiling-node": "^1.3.5",
+    "@sentry/nextjs": "^7.101.1",
+    "@sentry/profiling-node": "^7.101.1",
     "@sentry/types": "^7.88.0",
     "@t3-oss/env-nextjs": "^0.8.0",
     "@tailwindcss/forms": "^0.5.7",
     "@tanstack/react-query": "^4.36.1",
-    "@tanstack/react-table": "^8.11.7",
+    "@tanstack/react-table": "^8.11.8",
     "@tremor/react": "^3.11.1",
     "@trpc/client": "^10.45.0",
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
-    "@vercel/edge-config": "^0.4.1",
     "bcryptjs": "^2.4.3",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
     "cmdk": "^0.2.1",
-    "core-js": "^3.35.1",
+    "core-js": "^3.36.0",
     "cors": "^2.8.5",
     "date-fns": "^3.3.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
     "js-tiktoken": "^1.0.10",
     "lodash": "^4.17.21",
-    "lucide-react": "^0.321.0",
+    "lucide-react": "^0.330.0",
     "next": "^14.1.0",
-    "next-auth": "^4.24.5",
+    "next-auth": "^4.24.6",
     "next-query-params": "^5.0.0",
     "nodemailer": "^6.9.9",
-    "posthog-js": "^1.103.1",
-    "posthog-node": "^3.6.1",
+    "posthog-js": "^1.105.9",
+    "posthog-node": "^3.6.3",
     "react": "18.2.0",
     "react-day-picker": "^8.10.0",
     "react-dom": "18.2.0",
-    "react-hook-form": "^7.49.3",
+    "react-hook-form": "^7.50.1",
     "react-icons": "^5.0.1",
     "react-responsive": "^9.0.2",
     "react18-json-view": "^0.2.7",
+    "sonner": "^1.4.0",
     "superjson": "2.2.1",
     "tailwind-merge": "^2.2.1",
     "tailwindcss-animate": "^1.0.7",
@@ -107,7 +110,7 @@
     "@mermaid-js/mermaid-cli": "^10.7.0",
     "@playwright/test": "^1.41.2",
     "@release-it/bumper": "^6.0.1",
-    "@testing-library/jest-dom": "^6.4.1",
+    "@testing-library/jest-dom": "^6.4.2",
     "@testing-library/react": "^14.2.1",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
@@ -116,27 +119,27 @@
     "@types/lodash": "^4.14.202",
     "@types/node": "20.10.5",
     "@types/nodemailer": "^6.4.14",
-    "@types/react": "^18.2.51",
-    "@types/react-dom": "^18.2.18",
+    "@types/react": "^18.2.55",
+    "@types/react-dom": "^18.2.19",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/eslint-plugin": "^6.20.0",
-    "@typescript-eslint/parser": "^6.20.0",
+    "@typescript-eslint/eslint-plugin": "^6.21.0",
+    "@typescript-eslint/parser": "^6.21.0",
     "autoprefixer": "^10.4.17",
     "dotenv-cli": "^7.3.0",
     "eslint": "^8.56.0",
     "eslint-config-next": "^14.1.0",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
-    "postcss": "^8.4.33",
-    "prettier": "^3.2.4",
+    "postcss": "^8.4.35",
+    "prettier": "^3.2.5",
     "prettier-plugin-tailwindcss": "^0.5.11",
-    "prisma": "^5.8.1",
+    "prisma": "^5.9.1",
     "prisma-erd-generator": "^1.11.2",
-    "release-it": "^17.0.3",
+    "release-it": "^17.0.5",
     "tailwindcss": "^3.4.1",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "tsx": "^4.7.0",
+    "tsx": "^4.7.1",
     "typescript": "^5.3.3"
   },
   "ct3aMetadata": {

prisma/migrations/20240203184148_update_pricing/migration.sql

@@ -0,0 +1,27 @@
+-- This is an empty migration.
+
+DELETE FROM models
+WHERE id in ('clrkwk4cb000308l5go4b6otm', 'clrntjt89000a08jw0gcdbd5a');
+
+
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- https://openai.com/blog/new-embedding-models-and-api-updates
+  
+
+  -- fix prices
+  ('clrkwk4cb000308l5go4b6otm', NULL, 'gpt-3.5-turbo-16k', '(?i)^(gpt-)(35|3.5)(-turbo-16k)$', NULL, 0.000003, 0.000004, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-16k" }'),
+  ('clrntjt89000a08jw0gcdbd5a', NULL, 'gpt-3.5-turbo-16k-0613', '(?i)^(gpt-)(35|3.5)(-turbo-16k-0613)$', NULL, 0.000003, 0.000004, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-16k-0613" }')

prisma/migrations/20240212175433_add_audit_log_table/migration.sql

@@ -0,0 +1,28 @@
+-- CreateTable
+CREATE TABLE "audit_logs" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "user_id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "user_project_role" "MembershipRole" NOT NULL,
+    "resource_type" TEXT NOT NULL,
+    "resource_id" TEXT NOT NULL,
+    "action" TEXT NOT NULL,
+    "before" TEXT,
+    "after" TEXT,
+
+    CONSTRAINT "audit_logs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "audit_logs_project_id_idx" ON "audit_logs"("project_id");
+
+-- CreateIndex
+CREATE INDEX "audit_logs_created_at_idx" ON "audit_logs"("created_at");
+
+-- AddForeignKey
+ALTER TABLE "audit_logs" ADD CONSTRAINT "audit_logs_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "audit_logs" ADD CONSTRAINT "audit_logs_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/migrations/20240213124148_update_openai_pricing/migration.sql

@@ -0,0 +1,25 @@
+-- This is an empty migration.
+
+DELETE FROM models
+WHERE id in ('clruwnahl00040al78f1lb0at');
+
+
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  -- according to email, gpt-3.5-turbo and gpt-3.5-turbo-16k will point to 0125 models as of 2024-02-16
+  -- gpt-3.5-turbo-0125 now supports 16k token length. 16k model will point to regular 3.5 turbo model according to mail.
+  ('clruwnahl00040al78f1lb0at', NULL, 'gpt-3.5-turbo', '(?i)^(gpt-)(35|3.5)(-turbo)$', '2024-02-16', 0.0000005, 0.0000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo" }'),
+  ('clsk9lntu000008jwfc51bbqv', NULL, 'gpt-3.5-turbo-16k', '(?i)^(gpt-)(35|3.5)(-turbo-16k)$', '2024-02-16', 0.0000005, 0.0000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-3.5-turbo-16k" }')

prisma/migrations/20240214232619_prompts_add_indicies/migration.sql

@@ -0,0 +1,5 @@
+-- CreateIndex
+CREATE INDEX "prompts_project_id_id_idx" ON "prompts"("project_id", "id");
+
+-- CreateIndex
+CREATE INDEX "prompts_project_id_idx" ON "prompts"("project_id");

prisma/migrations/20240215224148_update_openai_pricing/migration.sql

@@ -0,0 +1,17 @@
+-- This is an empty migration.
+
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  ('clsnq07bn000008l4e46v1ll8', NULL, 'gpt-4-turbo-preview', '(?i)^(gpt-4-turbo-preview)$', '2023-11-06', 0.00001, 0.00003, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4" }')

prisma/migrations/20240215234937_fix_observations_view/migration.sql

@@ -0,0 +1,50 @@
+CREATE OR REPLACE VIEW "observations_view" AS
+SELECT
+    o.*,
+    m.id AS "model_id",
+    m.start_date AS "model_start_date",
+    m.input_price,
+    m.output_price,
+    m.total_price,
+    m.tokenizer_config AS "tokenizer_config",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.prompt_tokens::decimal * m.input_price
+        ELSE
+            o.input_cost
+    END AS "calculated_input_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            o.completion_tokens::decimal * m.output_price
+        ELSE
+            o.output_cost
+    END AS "calculated_output_cost",
+    CASE 
+        WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
+            CASE 
+                WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
+                    m.total_price * o.total_tokens
+                ELSE
+                    o.prompt_tokens::decimal * m.input_price + 
+                    o.completion_tokens::decimal * m.output_price
+            END
+        ELSE
+            o.total_cost
+    END AS "calculated_total_cost",
+    CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency"
+FROM
+    observations o
+LEFT JOIN models m ON m.id = (
+    SELECT
+        id
+    FROM
+        models
+    WHERE (project_id = o.project_id OR project_id IS NULL)
+    AND model_name = o.internal_model
+    AND (start_date < o.start_time OR start_date is NULL)
+    AND o.unit::TEXT = unit
+    ORDER BY
+        project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
+        start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
+    LIMIT 1
+)

prisma/schema.prisma

@@ -72,6 +72,7 @@ model User {
     createdAt     DateTime               @default(now()) @map("created_at")
     updatedAt     DateTime               @default(now()) @updatedAt @map("updated_at")
     featureFlags  String[]               @default([]) @map("feature_flags")
+    AuditLog      AuditLog[]
 
     @@map("users")
 }
@@ -101,6 +102,7 @@ model Project {
     sessions     TraceSession[]
     Prompt       Prompt[]
     Model        Model[]
+    AuditLog     AuditLog[]
 
     @@map("projects")
 }
@@ -423,6 +425,8 @@ model Prompt {
 
     @@unique([projectId, name, version])
     @@index([projectId, name, version])
+    @@index([projectId, id])
+    @@index([projectId])
     @@map("prompts")
 }
 
@@ -440,7 +444,7 @@ model Model {
     inputPrice      Decimal?  @map("input_price")
     outputPrice     Decimal?  @map("output_price")
     totalPrice      Decimal?  @map("total_price")
-    unit            String // TOKENS or CHARACTERS
+    unit            String // TOKENS, CHARACTERS, MILLISECONDS, SECONDS, or IMAGES
     tokenizerId     String?   @map("tokenizer_id")
     tokenizerConfig Json?     @map("tokenizer_config")
 
@@ -489,6 +493,27 @@ view ObservationView {
     calculatedInputCost  Decimal? @map("calculated_input_cost")
     calculatedOutputCost Decimal? @map("calculated_output_cost")
     calculatedTotalCost  Decimal? @map("calculated_total_cost")
+    latency              Decimal? @map("latency")
 
     @@map("observations_view")
 }
+
+model AuditLog {
+    id              String         @id @default(cuid())
+    createdAt       DateTime       @default(now()) @map("created_at")
+    updatedAt       DateTime       @default(now()) @updatedAt @map("updated_at")
+    userId          String         @map("user_id")
+    user            User           @relation(fields: [userId], references: [id], onDelete: Cascade)
+    projectId       String         @map("project_id")
+    project         Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    userProjectRole MembershipRole @map("user_project_role")
+    resourceType    String         @map("resource_type")
+    resourceId      String         @map("resource_id")
+    action          String
+    before          String? //stringified JSON
+    after           String? // stringified JSON
+
+    @@index([projectId])
+    @@index([createdAt])
+    @@map("audit_logs")
+}

prisma/seed.ts

@@ -5,6 +5,7 @@ import {
 } from "@/src/features/public-api/lib/apiKeys";
 import { hash } from "bcryptjs";
 import { parseArgs } from "node:util";
+import { ModelUsageUnit } from "@/src/constants";
 
 const options = {
   environment: { type: "string" },
@@ -487,7 +488,7 @@ async function main() {
                   ? { prompt: { connect: { id: prompt.id } } }
                   : {}),
               },
-              unit: "TOKENS",
+              unit: ModelUsageUnit.Tokens,
             },
           });
           if (Math.random() > 0.6)

src/__tests__/cost.servertest.ts

@@ -1,6 +1,7 @@
 /** @jest-environment node */
 
 import { pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
 import { prisma } from "@/src/server/db";
 
 describe("cost retrieval tests", () => {
@@ -111,7 +112,7 @@ describe("cost retrieval tests", () => {
           projectId: null,
           startDate: new Date("2023-12-01"),
           tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
         },
       });
 
@@ -130,7 +131,7 @@ describe("cost retrieval tests", () => {
           model: "gpt-3.5-turbo",
           internalModel: "gpt-3.5-turbo",
           startTime: new Date("2024-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           promptTokens: input.promptTokens,
           completionTokens: input.completionTokens,
           totalTokens: input.totalTokens,
@@ -181,7 +182,7 @@ describe("cost retrieval tests", () => {
           projectId: null,
           startDate: new Date("2023-12-01"),
           tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
         },
       });
       await prisma.model.create({
@@ -194,7 +195,7 @@ describe("cost retrieval tests", () => {
           matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
           startDate: new Date("2023-12-01"),
           tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
         },
       });
@@ -214,7 +215,7 @@ describe("cost retrieval tests", () => {
           model: "gpt-3.5-turbo",
           internalModel: "gpt-3.5-turbo",
           startTime: new Date("2024-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           promptTokens: 200,
           completionTokens: 3000,
           totalTokens: undefined,
@@ -238,8 +239,23 @@ describe("cost retrieval tests", () => {
       );
     });
   });
+
   it(`should prioritize latest models`, async () => {
     await pruneDatabase();
+    await prisma.model.create({
+      data: {
+        id: "model-0",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000000",
+        outputPrice: "0.0000000",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        projectId: null,
+        startDate: null,
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
 
     await prisma.model.create({
       data: {
@@ -252,7 +268,7 @@ describe("cost retrieval tests", () => {
         projectId: null,
         startDate: new Date("2023-12-01"),
         tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
       },
     });
     await prisma.model.create({
@@ -265,7 +281,7 @@ describe("cost retrieval tests", () => {
         matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
         startDate: new Date("2023-12-02"),
         tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
       },
     });
 
@@ -284,7 +300,7 @@ describe("cost retrieval tests", () => {
         model: "gpt-3.5-turbo",
         internalModel: "gpt-3.5-turbo",
         startTime: new Date("2024-01-01T00:00:00.000Z"),
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
         promptTokens: 200,
         completionTokens: 3000,
         totalTokens: undefined,
@@ -304,34 +320,35 @@ describe("cost retrieval tests", () => {
     expect(view?.calculatedTotalCost?.toString()).toBe("0.0124");
   });
 
-  it(`should prioritize old model if the latest model is not own one`, async () => {
+  it(`should prioritize own models`, async () => {
     await pruneDatabase();
+    await prisma.model.create({
+      data: {
+        id: "model-0",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000000",
+        outputPrice: "0.0000000",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        projectId: null,
+        startDate: null,
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
 
     await prisma.model.create({
       data: {
         id: "model-1",
         modelName: "gpt-3.5-turbo",
-        inputPrice: "0.0010",
-        outputPrice: "0.0020",
+        inputPrice: "0.0000010",
+        outputPrice: "0.0000020",
         totalPrice: "0.1",
         matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
-        startDate: new Date("2023-12-02"),
-        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
-        unit: "TOKENS",
-      },
-    });
-    await prisma.model.create({
-      data: {
-        id: "model-2",
-        modelName: "gpt-3.5-turbo",
-        inputPrice: "0.0000020",
-        outputPrice: "0.0000040",
-        totalPrice: undefined,
-        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
-        startDate: new Date("2023-12-01"),
-        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        startDate: null,
         project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
-        unit: "TOKENS",
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
       },
     });
 
@@ -350,7 +367,136 @@ describe("cost retrieval tests", () => {
         model: "gpt-3.5-turbo",
         internalModel: "gpt-3.5-turbo",
         startTime: new Date("2024-01-01T00:00:00.000Z"),
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
+        promptTokens: 200,
+        completionTokens: 3000,
+        totalTokens: undefined,
+      },
+    });
+
+    const view = await prisma.observationView.findFirst({
+      where: { traceId: dbTrace.id },
+    });
+
+    console.log(view);
+
+    // calculated cost fields
+    expect(view?.modelId).toBe("model-1");
+  });
+
+  it(`should prioritize old model if the latest model is not own one`, async () => {
+    await pruneDatabase();
+
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0010",
+        outputPrice: "0.0020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-02"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000020",
+        outputPrice: "0.0000040",
+        totalPrice: undefined,
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-01"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+
+    const dbTrace = await prisma.trace.create({
+      data: {
+        name: "trace-name",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+      },
+    });
+
+    await prisma.observation.create({
+      data: {
+        traceId: dbTrace.id,
+        type: "GENERATION",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        model: "gpt-3.5-turbo",
+        internalModel: "gpt-3.5-turbo",
+        startTime: new Date("2024-01-01T00:00:00.000Z"),
+        unit: ModelUsageUnit.Tokens,
+        promptTokens: 200,
+        completionTokens: 3000,
+        totalTokens: undefined,
+      },
+    });
+
+    const view = await prisma.observationView.findFirst({
+      where: { traceId: dbTrace.id },
+    });
+
+    console.log(view);
+
+    // calculated cost fields
+    expect(view?.modelId).toBe("model-2");
+    expect(view?.calculatedInputCost?.toString()).toBe("0.0004");
+    expect(view?.calculatedOutputCost?.toString()).toBe("0.012");
+    expect(view?.calculatedTotalCost?.toString()).toBe("0.0124");
+  });
+
+  it(`should prioritize new model if the latest model is own one`, async () => {
+    await pruneDatabase();
+
+    await prisma.model.create({
+      data: {
+        id: "model-1",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0010",
+        outputPrice: "0.0020",
+        totalPrice: "0.1",
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: null,
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+    await prisma.model.create({
+      data: {
+        id: "model-2",
+        modelName: "gpt-3.5-turbo",
+        inputPrice: "0.0000020",
+        outputPrice: "0.0000040",
+        totalPrice: undefined,
+        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
+        startDate: new Date("2023-12-01"),
+        tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        unit: ModelUsageUnit.Tokens,
+      },
+    });
+
+    const dbTrace = await prisma.trace.create({
+      data: {
+        name: "trace-name",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+      },
+    });
+
+    await prisma.observation.create({
+      data: {
+        traceId: dbTrace.id,
+        type: "GENERATION",
+        project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
+        model: "gpt-3.5-turbo",
+        internalModel: "gpt-3.5-turbo",
+        startTime: new Date("2024-01-01T00:00:00.000Z"),
+        unit: ModelUsageUnit.Tokens,
         promptTokens: 200,
         completionTokens: 3000,
         totalTokens: undefined,
@@ -383,7 +529,7 @@ describe("cost retrieval tests", () => {
         matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
         startDate: new Date("2023-12-02"),
         tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
       },
     });
     await prisma.model.create({
@@ -397,7 +543,7 @@ describe("cost retrieval tests", () => {
         startDate: new Date("2023-12-01"),
         tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
         project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
       },
     });
 
@@ -416,7 +562,7 @@ describe("cost retrieval tests", () => {
         model: "gpt-3.5-turbo",
         internalModel: "gpt-3.5-turbo",
         startTime: new Date("2024-01-01T00:00:00.000Z"),
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
         promptTokens: 200,
         completionTokens: 3000,
         totalTokens: undefined,

src/__tests__/generation-exports.servertest.ts

@@ -0,0 +1,194 @@
+/** @jest-environment node */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+
+import { pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
+import { appRouter } from "@/src/server/api/root";
+import { createInnerTRPCContext } from "@/src/server/api/trpc";
+import { prisma } from "@/src/server/db";
+import type { Session } from "next-auth";
+
+describe("observations.export RPC", () => {
+  const numberOfGenerations = 5;
+  const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+  beforeAll(async () => {
+    // Disable S3 upload
+    process.env.S3_ENDPOINT = "";
+
+    await pruneDatabase();
+    const traceId = "trace-1";
+
+    await prisma.trace.create({
+      data: {
+        id: traceId,
+        name: "trace-name",
+        userId: "user-1",
+        projectId,
+        metadata: { key: "value" },
+        release: "1.0.0",
+        version: "2.0.0",
+      },
+    });
+
+    for (let i = 1; i <= numberOfGenerations; i++) {
+      await prisma.observation.create({
+        data: {
+          type: "GENERATION",
+          id: `generation-${i}`,
+          name: `generation-${i}`,
+          model: "gpt-3.5-turbo",
+          totalCost: 1,
+          startTime: new Date("2021-01-01T00:00:00.000Z"),
+          endTime: new Date("2021-01-01T00:00:05.000Z"),
+          project: { connect: { id: projectId } },
+          traceId,
+          input: [
+            {
+              role: "system",
+              content: "Be a helpful assistant",
+            },
+            {
+              role: "user",
+              content: "How can i create a React component?",
+            },
+          ],
+          output: {
+            completion: `Creating a React component can be done in two ways.`,
+          },
+          metadata: {
+            user: `user-@langfuse.com`,
+          },
+          unit: ModelUsageUnit.Tokens,
+        },
+      });
+    }
+  });
+
+  afterAll(async () => await pruneDatabase());
+
+  const session: Session = {
+    expires: "1",
+    user: {
+      id: "clgb17vnp000008jjere5g15i",
+      name: "John Doe",
+      projects: [
+        {
+          id: projectId,
+          role: "ADMIN",
+          name: "test",
+        },
+      ],
+      featureFlags: {
+        templateFlag: true,
+      },
+      admin: true,
+    },
+  };
+
+  const ctx = createInnerTRPCContext({ session });
+  const caller = appRouter.createCaller({ ...ctx, prisma });
+
+  it("should return a CSV file", async () => {
+    const result = await caller.generations.export({
+      fileFormat: "CSV",
+      orderBy: { column: "id", order: "ASC" },
+      filter: [
+        {
+          column: "start_time",
+          type: "datetime",
+          operator: ">",
+          value: new Date("1990-01-01"),
+        },
+      ],
+      projectId,
+      searchQuery: null,
+    });
+
+    if (result.type !== "data")
+      throw new Error("No data returned. Is S3 accidentally enabled?");
+    const { data, fileName } = result;
+
+    const fileExtension = fileName.split(".").pop();
+    expect(fileName).toContain(`lf-export-${projectId}`);
+    expect(fileExtension).toBe("csv");
+    expect(data.split("\n").filter(Boolean).length).toBe(
+      numberOfGenerations + 1,
+    );
+  });
+
+  it("should return a JSON file", async () => {
+    const result = await caller.generations.export({
+      fileFormat: "JSON",
+      orderBy: { column: "id", order: "ASC" },
+      filter: [
+        {
+          column: "start_time",
+          type: "datetime",
+          operator: ">",
+          value: new Date("1990-01-01"),
+        },
+      ],
+      projectId,
+      searchQuery: null,
+    });
+
+    if (result.type !== "data")
+      throw new Error("No data returned. Is S3 accidentally enabled?");
+    const { data, fileName } = result;
+
+    const fileExtension = fileName.split(".").pop();
+    expect(fileName).toContain(`lf-export-${projectId}`);
+    expect(fileExtension).toBe("json");
+
+    expect(JSON.parse(data).length).toBe(numberOfGenerations);
+  });
+
+  it("should return a OPENAI-JSONL file", async () => {
+    const result = await caller.generations.export({
+      fileFormat: "OPENAI-JSONL",
+      orderBy: { column: "id", order: "ASC" },
+      filter: [
+        {
+          column: "start_time",
+          type: "datetime",
+          operator: ">",
+          value: new Date("1990-01-01"),
+        },
+      ],
+      projectId,
+      searchQuery: null,
+    });
+
+    if (result.type !== "data")
+      throw new Error("No data returned. Is S3 accidentally enabled?");
+    const { data, fileName } = result;
+
+    const fileExtension = fileName.split(".").pop();
+    expect(fileName).toContain(`lf-export-${projectId}`);
+    expect(fileExtension).toBe("jsonl");
+
+    expect(data.split("\n").filter(Boolean).length).toBe(numberOfGenerations);
+  });
+
+  it("should throw on unsupported file formats", async () => {
+    const unsupportedFileFormat = "XLSX";
+
+    const call = caller.generations.export({
+      fileFormat: unsupportedFileFormat as unknown as "JSON",
+      orderBy: { column: "id", order: "ASC" },
+      filter: [
+        {
+          column: "start_time",
+          type: "datetime",
+          operator: ">",
+          value: new Date("1990-01-01"),
+        },
+      ],
+      projectId,
+      searchQuery: null,
+    });
+
+    await expect(call).rejects.toThrow();
+  });
+});

src/__tests__/generations.servertest.ts

@@ -1,9 +1,11 @@
 /** @jest-environment node */
 
-import { prisma } from "@/src/server/db";
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
 
+import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
+import { prisma } from "@/src/server/db";
+
 describe("/api/public/generations API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
 
@@ -13,9 +15,9 @@ describe("/api/public/generations API Endpoint", () => {
         input: 100,
         output: 200,
         total: 100,
-        unit: "CHARACTERS",
+        unit: ModelUsageUnit.Characters,
       },
-      expectedUnit: "CHARACTERS",
+      expectedUnit: ModelUsageUnit.Characters,
       expectedPromptTokens: 100,
       expectedCompletionTokens: 200,
       expectedTotalTokens: 100,
@@ -23,9 +25,9 @@ describe("/api/public/generations API Endpoint", () => {
     {
       usage: {
         total: 100,
-        unit: "CHARACTERS",
+        unit: ModelUsageUnit.Characters,
       },
-      expectedUnit: "CHARACTERS",
+      expectedUnit: ModelUsageUnit.Characters,
       expectedPromptTokens: 0,
       expectedCompletionTokens: 0,
       expectedTotalTokens: 100,
@@ -48,7 +50,7 @@ describe("/api/public/generations API Endpoint", () => {
       expectedPromptTokens: 100,
       expectedCompletionTokens: 200,
       expectedTotalTokens: 100,
-      expectedUnit: "TOKENS",
+      expectedUnit: ModelUsageUnit.Tokens,
     },
     {
       usage: {
@@ -57,7 +59,7 @@ describe("/api/public/generations API Endpoint", () => {
       expectedPromptTokens: 0,
       expectedCompletionTokens: 0,
       expectedTotalTokens: 100,
-      expectedUnit: "TOKENS",
+      expectedUnit: ModelUsageUnit.Tokens,
     },
     {
       usage: undefined,

src/__tests__/ingestion.servertest.ts

@@ -1,9 +1,11 @@
 /** @jest-environment node */
 
+import { v4 } from "uuid";
+
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
 import { cleanEvent } from "@/src/pages/api/public/ingestion";
 import { prisma } from "@/src/server/db";
-import { v4 } from "uuid";
 
 describe("/api/public/ingestion API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -14,12 +16,12 @@ describe("/api/public/ingestion API Endpoint", () => {
         input: 100,
         output: 200,
         total: 100,
-        unit: "CHARACTERS",
+        unit: ModelUsageUnit.Characters,
         inputCost: 123,
         outputCost: 456,
         totalCost: 789,
       },
-      expectedUnit: "CHARACTERS",
+      expectedUnit: ModelUsageUnit.Characters,
       expectedPromptTokens: 100,
       expectedCompletionTokens: 200,
       expectedTotalTokens: 100,
@@ -27,13 +29,45 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       usage: {
         total: 100,
-        unit: "CHARACTERS",
+        unit: ModelUsageUnit.Characters,
       },
-      expectedUnit: "CHARACTERS",
+      expectedUnit: ModelUsageUnit.Characters,
       expectedPromptTokens: 0,
       expectedCompletionTokens: 0,
       expectedTotalTokens: 100,
     },
+    {
+      usage: {
+        total: 100,
+        unit: ModelUsageUnit.Milliseconds,
+      },
+      expectedUnit: ModelUsageUnit.Milliseconds,
+      expectedPromptTokens: 0,
+      expectedCompletionTokens: 0,
+      expectedTotalTokens: 100,
+    },
+    {
+      usage: {
+        input: 1,
+        output: 2,
+        unit: ModelUsageUnit.Images,
+      },
+      expectedUnit: ModelUsageUnit.Images,
+      expectedPromptTokens: 1,
+      expectedCompletionTokens: 2,
+      expectedTotalTokens: 3,
+    },
+    {
+      usage: {
+        input: 30,
+        output: 10,
+        unit: ModelUsageUnit.Seconds,
+      },
+      expectedUnit: ModelUsageUnit.Seconds,
+      expectedPromptTokens: 30,
+      expectedCompletionTokens: 10,
+      expectedTotalTokens: 40,
+    },
     {
       usage: {
         total: 100,
@@ -52,7 +86,7 @@ describe("/api/public/ingestion API Endpoint", () => {
       expectedPromptTokens: 100,
       expectedCompletionTokens: 200,
       expectedTotalTokens: 100,
-      expectedUnit: "TOKENS",
+      expectedUnit: ModelUsageUnit.Tokens,
     },
     {
       usage: {
@@ -61,7 +95,7 @@ describe("/api/public/ingestion API Endpoint", () => {
       expectedPromptTokens: 0,
       expectedCompletionTokens: 0,
       expectedTotalTokens: 100,
-      expectedUnit: "TOKENS",
+      expectedUnit: ModelUsageUnit.Tokens,
     },
     {
       usage: undefined,
@@ -173,8 +207,6 @@ describe("/api/public/ingestion API Endpoint", () => {
 
       expect(response.status).toBe(207);
 
-      console.log("response body", response.body);
-
       const dbTrace = await prisma.trace.findMany({
         where: {
           name: "trace-name",
@@ -256,7 +288,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       observationExternalModel: "gpt-3.5",
       observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
-      modelUnit: "TOKENS",
+      modelUnit: ModelUsageUnit.Tokens,
       expectedInternalModel: "gpt-3.5-turbo",
       expectedPromptTokens: 5,
       expectedCompletionTokens: 7,
@@ -265,7 +297,7 @@ describe("/api/public/ingestion API Endpoint", () => {
           modelName: "gpt-3.5-turbo",
           matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
           startDate: new Date("2021-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
       ],
@@ -273,7 +305,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       observationExternalModel: "gpt-3.5",
       observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
-      modelUnit: "TOKENS",
+      modelUnit: ModelUsageUnit.Tokens,
       expectedInternalModel: "gpt-3.5-turbo",
       expectedPromptTokens: 5,
       expectedCompletionTokens: 7,
@@ -282,7 +314,7 @@ describe("/api/public/ingestion API Endpoint", () => {
           modelName: "gpt-3.5-turbo",
           matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
           startDate: null,
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
       ],
@@ -290,7 +322,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       observationExternalModel: "GPT-3.5",
       observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
-      modelUnit: "TOKENS",
+      modelUnit: ModelUsageUnit.Tokens,
       expectedInternalModel: "gpt-3.5-turbo",
       expectedPromptTokens: 5,
       expectedCompletionTokens: 7,
@@ -299,7 +331,7 @@ describe("/api/public/ingestion API Endpoint", () => {
           modelName: "gpt-3.5-turbo",
           matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
           startDate: new Date("2021-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
       ],
@@ -307,7 +339,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       observationExternalModel: "GPT-3.5",
       observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
-      modelUnit: "TOKENS",
+      modelUnit: ModelUsageUnit.Tokens,
       expectedInternalModel: "gpt-3.5-turbo",
       expectedPromptTokens: 5,
       expectedCompletionTokens: 7,
@@ -316,14 +348,14 @@ describe("/api/public/ingestion API Endpoint", () => {
           modelName: "gpt-3.5-turbo",
           matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
           startDate: new Date("2021-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
         {
           modelName: "gpt-3.5-turbo-new",
           matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
           startDate: new Date("2021-01-01T10:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
       ],
@@ -331,7 +363,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       observationExternalModel: "GPT-3.5",
       observationStartTime: new Date("2021-01-02T00:00:00.000Z"),
-      modelUnit: "TOKENS",
+      modelUnit: ModelUsageUnit.Tokens,
       expectedInternalModel: "gpt-3.5-turbo",
       expectedPromptTokens: 5,
       expectedCompletionTokens: 7,
@@ -340,14 +372,14 @@ describe("/api/public/ingestion API Endpoint", () => {
           modelName: "gpt-3.5-turbo-new",
           matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
           startDate: new Date("2021-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
         {
           modelName: "gpt-3.5-turbo",
           matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
           startDate: new Date("2021-01-01T10:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
           tokenizerModel: "gpt-3.5-turbo",
         },
@@ -356,7 +388,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       observationExternalModel: "ft:gpt-3.5-turbo-1106:my-org:custom_suffix:id",
       observationStartTime: new Date("2022-01-01T10:00:00.000Z"),
-      modelUnit: "TOKENS",
+      modelUnit: ModelUsageUnit.Tokens,
       expectedInternalModel: "ft:gpt-3.5-turbo-1106",
       expectedPromptTokens: 0,
       expectedCompletionTokens: 0,
@@ -365,7 +397,7 @@ describe("/api/public/ingestion API Endpoint", () => {
           modelName: "ft:gpt-3.5-turbo-1106",
           matchPattern: "(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$",
           startDate: new Date("2022-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
       ],
@@ -373,7 +405,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       observationExternalModel: "ft:babbage-002:my-org#2:custom_suffix-2:id",
       observationStartTime: new Date("2022-01-01T10:00:00.000Z"),
-      modelUnit: "TOKENS",
+      modelUnit: ModelUsageUnit.Tokens,
       expectedInternalModel: "ft:babbage-002",
       expectedPromptTokens: 0,
       expectedCompletionTokens: 0,
@@ -382,7 +414,7 @@ describe("/api/public/ingestion API Endpoint", () => {
           modelName: "ft:babbage-002",
           matchPattern: "(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$",
           startDate: new Date("2022-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
       ],
@@ -390,7 +422,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       observationExternalModel: "GPT-4",
       observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
-      modelUnit: "TOKENS",
+      modelUnit: ModelUsageUnit.Tokens,
       expectedInternalModel: null,
       expectedPromptTokens: 0,
       expectedCompletionTokens: 0,
@@ -399,7 +431,7 @@ describe("/api/public/ingestion API Endpoint", () => {
           modelName: "gpt-3.5-turbo",
           matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
           startDate: new Date("2021-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
       ],
@@ -407,7 +439,7 @@ describe("/api/public/ingestion API Endpoint", () => {
     {
       observationExternalModel: "GPT-3",
       observationStartTime: new Date("2021-01-01T00:00:00.000Z"),
-      modelUnit: "CHARACTERS",
+      modelUnit: ModelUsageUnit.Characters,
       expectedInternalModel: null,
       expectedPromptTokens: 0,
       expectedCompletionTokens: 0,
@@ -416,7 +448,7 @@ describe("/api/public/ingestion API Endpoint", () => {
           modelName: "gpt-3.5-turbo",
           matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
           startDate: new Date("2021-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           tokenizerId: "openai",
         },
       ],
@@ -761,10 +793,10 @@ describe("/api/public/ingestion API Endpoint", () => {
 
     expect("errors" in responseOne.body).toBe(true);
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body?.errors.length).toBe(1);
+    expect(responseOne.body.errors.length).toBe(1);
     expect("successes" in responseOne.body).toBe(true);
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body?.successes.length).toBe(1);
+    expect(responseOne.body.successes.length).toBe(1);
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -813,10 +845,10 @@ describe("/api/public/ingestion API Endpoint", () => {
 
     expect("errors" in responseOne.body).toBe(true);
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body?.errors.length).toBe(1);
+    expect(responseOne.body.errors.length).toBe(1);
     expect("successes" in responseOne.body).toBe(true);
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body?.successes.length).toBe(1);
+    expect(responseOne.body.successes.length).toBe(1);
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -863,10 +895,10 @@ describe("/api/public/ingestion API Endpoint", () => {
 
     expect("errors" in responseOne.body).toBe(true);
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body?.errors.length).toBe(1);
+    expect(responseOne.body.errors.length).toBe(1);
     expect("successes" in responseOne.body).toBe(true);
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    expect(responseOne.body?.successes.length).toBe(1);
+    expect(responseOne.body.successes.length).toBe(1);
 
     const dbTrace = await prisma.trace.findMany({
       where: {
@@ -886,7 +918,7 @@ describe("/api/public/ingestion API Endpoint", () => {
         modelName: "gpt-3.5",
         matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
         startDate: new Date("2021-01-01T00:00:00.000Z"),
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
         tokenizerId: "openai",
         tokenizerConfig: {
           tokensPerMessage: 3,
@@ -968,7 +1000,7 @@ describe("/api/public/ingestion API Endpoint", () => {
         modelName: "gpt-3.5",
         matchPattern: "(?i)^(gpt-)(35|3.5)(-turbo)?$",
         startDate: new Date("2021-01-01T00:00:00.000Z"),
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
         tokenizerId: "openai",
         tokenizerConfig: {
           tokensPerMessage: 3,
@@ -1380,4 +1412,355 @@ IB Home   /   . . .   /   News   /   News about the IB   /   Why ChatGPT is an o
       expect(cleanedEvent).toStrictEqual(expected);
     });
   });
+
+  it("should allow score ingestion via Basic auth", async () => {
+    const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+    const traceId = "trace_id";
+
+    const scoreId = "score_id";
+    const scoreEventId = "score_event_id";
+    const scoreName = "score-name";
+    const scoreValue = 100.5;
+
+    // Seed db with a trace to be scored
+    await prisma.trace.create({
+      data: {
+        id: traceId,
+        name: "trace-name",
+        project: { connect: { id: projectId } },
+      },
+    });
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: scoreEventId,
+          type: "score-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: scoreId,
+            name: scoreName,
+            value: scoreValue,
+            traceId: traceId,
+          },
+        },
+      ],
+    });
+
+    expect(response.status).toBe(207);
+    expect(response.body.successes.length).toBe(1);
+    expect(response.body.successes[0]?.id).toBe(scoreEventId);
+    expect(response.body.errors.length).toBe(0);
+
+    const dbScore = await prisma.score.findUnique({
+      where: {
+        id: scoreId,
+      },
+    });
+
+    expect(dbScore?.id).toBe(scoreId);
+    expect(dbScore?.traceId).toBe(traceId);
+    expect(dbScore?.name).toBe(scoreName);
+    expect(dbScore?.value).toBe(scoreValue);
+  });
+
+  it("should allow score ingestion via Bearer auth", async () => {
+    const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+    const traceId = "trace_id";
+    const bearerAuth = "Bearer pk-lf-1234567890";
+
+    const scoreId = "score_id";
+    const scoreEventId = "score_event_id";
+    const scoreName = "score-name";
+    const scoreValue = 100.5;
+
+    // Seed db with a trace to be scored
+    await prisma.trace.create({
+      data: {
+        id: traceId,
+        name: "trace-name",
+        project: { connect: { id: projectId } },
+      },
+    });
+
+    const response = await makeAPICall(
+      "POST",
+      "/api/public/ingestion",
+      {
+        batch: [
+          {
+            id: scoreEventId,
+            type: "score-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: scoreId,
+              name: scoreName,
+              value: scoreValue,
+              traceId: traceId,
+            },
+          },
+        ],
+      },
+      bearerAuth,
+    );
+
+    expect(response.status).toBe(207);
+    expect(response.body.successes.length).toEqual(1);
+    expect(response.body.successes[0]?.id).toBe(scoreEventId);
+    expect(response.body.errors.length).toBe(0);
+
+    const dbScore = await prisma.score.findUnique({
+      where: {
+        id: scoreId,
+      },
+    });
+
+    expect(dbScore?.id).toBe(scoreId);
+    expect(dbScore?.traceId).toBe(traceId);
+    expect(dbScore?.name).toBe(scoreName);
+    expect(dbScore?.value).toBe(scoreValue);
+  });
+
+  it("should throw an Auth error on Bearer Auth for all events that are NOT 'score-create'", async () => {
+    const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+    const traceId = "trace_id";
+    const bearerAuth = "Bearer pk-lf-1234567890";
+
+    const scoreId = "score_id";
+    const scoreEventId = "score_event_id";
+    const scoreName = "score-name";
+    const scoreValue = 100.5;
+
+    const generationId = v4();
+    const spanId = v4();
+
+    const anotherTraceId = "another_trace_id";
+
+    // Seed db with a trace to be scored
+    await prisma.trace.create({
+      data: {
+        id: traceId,
+        name: "trace-name",
+        project: { connect: { id: projectId } },
+      },
+    });
+
+    const response = await makeAPICall(
+      "POST",
+      "/api/public/ingestion",
+      {
+        metadata: {
+          sdk_verion: "1.0.0",
+          sdk_name: "python",
+        },
+        batch: [
+          {
+            id: v4(),
+            type: "trace-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: anotherTraceId,
+              name: "trace-name",
+              userId: "user-1",
+              metadata: { key: "value" },
+              release: "1.0.0",
+              version: "2.0.0",
+              tags: ["tag-1", "tag-2"],
+            },
+          },
+          {
+            id: v4(),
+            type: "observation-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: generationId,
+              traceId: traceId,
+              type: "GENERATION",
+              name: "generation-name",
+              startTime: "2021-01-01T00:00:00.000Z",
+              endTime: "2021-01-01T00:00:00.000Z",
+              modelParameters: { key: "value" },
+              input: { key: "value" },
+              metadata: { key: "value" },
+              version: "2.0.0",
+            },
+          },
+          {
+            id: v4(),
+            type: "observation-update",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: generationId,
+              type: "GENERATION",
+              output: { key: "this is a great gpt output" },
+            },
+          },
+          {
+            id: v4(),
+            type: "observation-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: spanId,
+              traceId: traceId,
+              type: "SPAN",
+              name: "span-name",
+              startTime: "2021-01-01T00:00:00.000Z",
+              endTime: "2021-01-01T00:00:00.000Z",
+              input: { input: "value" },
+              metadata: { meta: "value" },
+              version: "2.0.0",
+            },
+          },
+          {
+            id: scoreEventId,
+            type: "score-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: scoreId,
+              name: "score-name",
+              value: 100.5,
+              traceId: traceId,
+            },
+          },
+        ],
+      },
+      bearerAuth,
+    );
+
+    expect(response.status).toBe(207);
+    expect(response.body.successes.length).toEqual(1);
+    expect(response.body.successes[0]?.id).toEqual(scoreEventId);
+
+    expect(response.body.errors.length).toEqual(4);
+
+    const dbScore = await prisma.score.findUnique({
+      where: {
+        id: scoreId,
+      },
+    });
+
+    expect(await prisma.trace.count()).toBe(1);
+    expect(await prisma.trace.count({ where: { id: traceId } })).toBe(1);
+    expect(await prisma.observation.count()).toBe(0);
+
+    expect(dbScore?.id).toBe(scoreId);
+    expect(dbScore?.traceId).toBe(traceId);
+    expect(dbScore?.name).toBe(scoreName);
+    expect(dbScore?.value).toBe(scoreValue);
+  });
+
+  it("should error on Bearer Auth for a trace from different project", async () => {
+    const otherProjectId = "other_project_id";
+    const traceId = "trace_id";
+    const bearerAuth = "Bearer pk-lf-1234567890";
+
+    const scoreId = "score_id";
+    const scoreEventId = "score_event_id";
+    const scoreName = "score-name";
+    const scoreValue = 100.5;
+
+    // Seed db with a trace to be scored
+    try {
+      await prisma.project.create({
+        data: {
+          id: otherProjectId,
+          name: "another-project",
+        },
+      });
+
+      await prisma.trace.create({
+        data: {
+          id: traceId,
+          name: "trace-name",
+          project: { connect: { id: otherProjectId } },
+        },
+      });
+
+      const response = await makeAPICall(
+        "POST",
+        "/api/public/ingestion",
+        {
+          batch: [
+            {
+              id: scoreEventId,
+              type: "score-create",
+              timestamp: new Date().toISOString(),
+              body: {
+                id: scoreId,
+                name: scoreName,
+                value: scoreValue,
+                traceId: traceId,
+              },
+            },
+          ],
+        },
+        bearerAuth,
+      );
+
+      expect(response.status).toBe(207);
+      expect(response.body.successes.length).toBe(0);
+      expect(response.body).toHaveProperty("errors");
+      expect(response.body.errors.length).toEqual(1);
+      expect(response.body.errors[0]?.id).toEqual(scoreEventId);
+
+      expect(await prisma.trace.count()).toBe(1);
+      expect(await prisma.trace.count({ where: { id: traceId } })).toBe(1);
+
+      const dbScore = await prisma.score.findUnique({
+        where: {
+          id: scoreId,
+        },
+      });
+
+      expect(dbScore).toBeNull();
+    } finally {
+      await prisma.project.delete({ where: { id: otherProjectId } });
+    }
+  });
+
+  it("should error on Bearer Auth with a trace that does not exist", async () => {
+    const traceId = "trace_id";
+    const bearerAuth = "Bearer pk-lf-1234567890";
+
+    const scoreId = "score_id";
+    const scoreEventId = "score_event_id";
+    const scoreName = "score-name";
+    const scoreValue = 100.5;
+
+    const response = await makeAPICall(
+      "POST",
+      "/api/public/ingestion",
+      {
+        batch: [
+          {
+            id: scoreEventId,
+            type: "score-create",
+            timestamp: new Date().toISOString(),
+            body: {
+              id: scoreId,
+              name: scoreName,
+              value: scoreValue,
+              traceId: traceId,
+            },
+          },
+        ],
+      },
+      bearerAuth,
+    );
+
+    expect(response.status).toBe(207);
+    expect(response.body.successes.length).toBe(0);
+    expect(response.body.errors.length).toBe(1);
+    expect(response.body.errors[0]?.id).toBe(scoreEventId);
+
+    expect(await prisma.trace.count()).toBe(0);
+
+    const dbScore = await prisma.score.findUnique({
+      where: {
+        id: scoreId,
+      },
+    });
+
+    expect(dbScore).toBeNull();
+  });
 });

src/__tests__/metrics-api.servertest.ts

@@ -0,0 +1,94 @@
+import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { v4 as uuidv4 } from "uuid";
+
+describe("/api/public/metrics/daily API Endpoint", () => {
+  beforeEach(async () => await pruneDatabase());
+  afterEach(async () => await pruneDatabase());
+
+  it("should handle daily metrics correctly", async () => {
+    await pruneDatabase();
+
+    // Create traces with observations on different days
+    const traceId1 = uuidv4();
+    const traceId2 = uuidv4();
+    await makeAPICall("POST", "/api/public/traces", {
+      id: traceId1,
+      timestamp: "2021-01-01T00:00:00.000Z",
+      name: "trace-day-1",
+      userId: "user-daily-metrics",
+      projectId: "project-daily-metrics",
+    });
+    await makeAPICall("POST", "/api/public/traces", {
+      id: traceId2,
+      timestamp: "2021-01-02T00:00:00.000Z",
+      name: "trace-day-2",
+      userId: "user-daily-metrics",
+      projectId: "project-daily-metrics",
+    });
+
+    // Simulate observations with usage metrics on different days
+    await makeAPICall("POST", "/api/public/generations", {
+      traceId: traceId1,
+      model: "modelA",
+      usage: { input: 100, output: 200, total: 300 },
+      startTime: "2021-01-01T00:00:00.000Z",
+      endTime: "2021-01-01T00:01:00.000Z",
+    });
+    await makeAPICall("POST", "/api/public/generations", {
+      traceId: traceId2,
+      model: "modelB",
+      usage: { input: 333 },
+      startTime: "2021-01-02T00:00:00.000Z",
+      endTime: "2021-01-02T00:02:00.000Z",
+    });
+    await makeAPICall("POST", "/api/public/generations", {
+      traceId: traceId2,
+      model: "modelC",
+      usage: { input: 666, output: 777, totalCost: 1024.22 },
+      startTime: "2021-01-02T00:00:00.000Z",
+      endTime: "2021-01-02T00:04:00.000Z",
+    });
+
+    // Retrieve the daily metrics
+    const dailyMetricsResponse = await makeAPICall<{
+      data: Array<Record<string, unknown>>;
+    }>("GET", `/api/public/metrics/daily`);
+    const dailyMetricsData = dailyMetricsResponse.body.data;
+
+    // Check if the daily metrics are calculated correctly
+    expect(dailyMetricsData).toHaveLength(2); // Two days of data
+    if (!dailyMetricsData[0])
+      throw new Error("dailyMetricsData[0] is undefined");
+    expect(dailyMetricsData[0].date).toBe("2021-01-02"); // Latest date first
+    expect(dailyMetricsData[0].countTraces).toBe(1);
+    expect(dailyMetricsData[0].totalCost).toEqual(1024.22);
+    expect(dailyMetricsData[0].usage).toEqual([
+      {
+        model: "modelB",
+        inputUsage: 333,
+        outputUsage: 0,
+        totalUsage: 333,
+      },
+      {
+        model: "modelC",
+        inputUsage: 666,
+        outputUsage: 777,
+        totalUsage: 1443,
+      },
+    ]);
+
+    if (!dailyMetricsData[1])
+      throw new Error("dailyMetricsData[1] is undefined");
+    expect(dailyMetricsData[1].date).toBe("2021-01-01");
+    expect(dailyMetricsData[1].countTraces).toBe(1);
+    expect(dailyMetricsData[1].totalCost).toEqual(0);
+    expect(dailyMetricsData[1].usage).toEqual([
+      {
+        model: "modelA",
+        inputUsage: 100,
+        outputUsage: 200,
+        totalUsage: 300,
+      },
+    ]);
+  });
+});

src/__tests__/model-match.servertest.ts

@@ -2,6 +2,7 @@
 
 import { modelMatch } from "@/scripts/model-match";
 import { pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
 import { prisma } from "@/src/server/db";
 
 describe("model match", () => {
@@ -17,7 +18,7 @@ describe("model match", () => {
         totalPrice: "0.1",
         matchPattern: "(.*)(gpt-3.5-turbo)?(.*)",
         projectId: null,
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
         tokenizerConfig: {
           tokensPerMessage: 3,
           tokensPerName: 1,
@@ -35,7 +36,7 @@ describe("model match", () => {
         totalPrice: "0.1",
         matchPattern: "(.*)(claude-1.3)?(.*)",
         projectId: null,
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
         tokenizerId: "claude",
       },
     });
@@ -48,7 +49,7 @@ describe("model match", () => {
           projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           model: "gpt-3.5-turbo",
           startTime: new Date("2024-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           promptTokens: 200,
           completionTokens: 3000,
           input: "I am a prompt",
@@ -59,7 +60,7 @@ describe("model match", () => {
           projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           model: "claude-1.3",
           startTime: new Date("2024-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           input: "I am a prompt",
           output: "I am a completion",
         },
@@ -68,7 +69,7 @@ describe("model match", () => {
           projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
           model: "claude-1.3",
           startTime: new Date("2024-01-01T00:00:00.000Z"),
-          unit: "TOKENS",
+          unit: ModelUsageUnit.Tokens,
           input: "I am a prompt",
           output: "I am a completion",
         },
@@ -91,5 +92,9 @@ describe("model match", () => {
       expect(observation.promptTokens).toBeGreaterThan(0);
       expect(observation.completionTokens).toBeGreaterThan(0);
     });
-  });
+
+    // temporary fix: wait for 5 additional seconds to ensure that the model match is complete
+    // had issue with the test failing because the model match was not complete and logged to console
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+  }, 10000);
 });

src/__tests__/observations.servertest.ts

@@ -1,8 +1,10 @@
 /** @jest-environment node */
 
-import { prisma } from "@/src/server/db";
-import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
+
+import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
+import { ModelUsageUnit } from "@/src/constants";
+import { prisma } from "@/src/server/db";
 import { type ObservationView } from "@prisma/client";
 
 describe("/api/public/observations API Endpoint", () => {
@@ -35,7 +37,7 @@ describe("/api/public/observations API Endpoint", () => {
         totalPrice: "0.1",
         matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
         projectId: null,
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
       },
     });
 
@@ -59,7 +61,7 @@ describe("/api/public/observations API Endpoint", () => {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
         },
         internalModel: "gpt-3.5-turbo",
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
       },
     });
 
@@ -120,7 +122,7 @@ describe("/api/public/observations API Endpoint", () => {
         totalPrice: "0.1",
         matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
         projectId: null,
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
       },
     });
 
@@ -140,7 +142,7 @@ describe("/api/public/observations API Endpoint", () => {
         completionTokens: 20,
         totalTokens: 30,
         version: "2.0.0",
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
         type: "GENERATION",
         project: {
           connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },

src/__tests__/query-builder.servertest.ts

@@ -16,28 +16,28 @@ describe("Build valid SQL queries", () => {
       {
         table: "traces",
         values: ["project-id"],
-        strings: [' FROM  traces t  WHERE  t."project_id" = ', " ;"],
+        strings: [' FROM  traces t  WHERE  t."project_id" = ', ";"],
       } as const,
       {
         table: "traces_observations",
         values: ["project-id", "project-id"],
         strings: [
           ' FROM  traces t LEFT JOIN observations_view o ON t.id = o.trace_id  WHERE  t."project_id" = ',
-          '  AND o."project_id" = ',
-          " ;",
+          ' AND o."project_id" = ',
+          ";",
         ],
       } as const,
       {
         table: "observations",
         values: ["project-id"],
-        strings: [' FROM  observations_view o  WHERE  o."project_id" = ', " ;"],
+        strings: [' FROM  observations_view o  WHERE  o."project_id" = ', ";"],
       } as const,
       {
         table: "traces_scores",
         values: ["project-id"],
         strings: [
           ' FROM  traces t JOIN scores s ON t.id = s.trace_id  WHERE  t."project_id" = ',
-          " ;",
+          ";",
         ],
       } as const,
     ].forEach((prop) => {
@@ -133,7 +133,7 @@ describe("Build valid SQL queries", () => {
           ],
           select: [],
         }),
-      ).toThrow("Column unknown not found");
+      ).toThrow("Invalid filter column: unknown");
     });
 
     it("should not select an unknown column", () => {

src/__tests__/scores.servertest.ts

@@ -231,4 +231,43 @@ describe("/api/public/scores API Endpoint", () => {
     expect(dbScore?.comment).toBe("comment-updated");
     expect(dbScore?.observationId).toBe(dbGeneration[0]!.id);
   });
+
+  it("should delete a score", async () => {
+    await pruneDatabase();
+
+    const traceId = uuidv4();
+
+    await makeAPICall("POST", "/api/public/traces", {
+      id: traceId,
+    });
+
+    const scoreId = uuidv4();
+    const createScore = await makeAPICall("POST", "/api/public/scores", {
+      id: scoreId,
+      name: "score-name",
+      value: 100.5,
+      traceId: traceId,
+      comment: "comment",
+    });
+
+    expect(createScore.status).toBe(200);
+    const dbScore = await prisma.score.findUnique({
+      where: {
+        id: scoreId,
+      },
+    });
+    expect(dbScore?.id).toBe(scoreId);
+
+    const deleteScore = await makeAPICall(
+      "DELETE",
+      `/api/public/scores/${scoreId}`,
+    );
+    expect(deleteScore.status).toBe(200);
+    const deletedScore = await prisma.score.findUnique({
+      where: {
+        id: scoreId,
+      },
+    });
+    expect(deletedScore).toBeNull();
+  });
 });

src/__tests__/test-utils.ts

@@ -27,30 +27,45 @@ export function createBasicAuthHeader(
   return `Basic ${base64Credentials}`;
 }
 
-export async function makeAPICall(
+export type IngestionAPIResponse = {
+  errors: ErrorIngestion[];
+  successes: SuccessfulIngestion[];
+};
+
+export type SuccessfulIngestion = {
+  id: string;
+  status: number;
+};
+
+export type ErrorIngestion = {
+  id: string;
+  status: number;
+  message: string;
+  error: string;
+};
+
+export async function makeAPICall<T = IngestionAPIResponse>(
   method: "POST" | "GET" | "PUT" | "DELETE" | "PATCH",
   url: string,
   body?: unknown,
-) {
+  auth?: string,
+): Promise<{ body: T; status: number }> {
   const finalUrl = `http://localhost:3000/${url}`;
+  const authorization =
+    auth || createBasicAuthHeader("pk-lf-1234567890", "sk-lf-1234567890");
   const options = {
     method: method,
     headers: {
       Accept: "application/json",
       "Content-Type": "application/json;charset=UTF-8",
-      Authorization: createBasicAuthHeader(
-        "pk-lf-1234567890",
-        "sk-lf-1234567890",
-      ),
+      Authorization: authorization,
     },
-    // Conditionally include the body property if the method is not "GET"
     ...(method !== "GET" &&
       body !== undefined && { body: JSON.stringify(body) }),
   };
-  const a = await fetch(finalUrl, options);
-
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-  return { body: await a.json(), status: a.status };
+  const response = await fetch(finalUrl, options);
+  const responseBody = (await response.json()) as T;
+  return { body: responseBody, status: response.status };
 }
 
 export const setupUserAndProject = async () => {

src/__tests__/traces.servertest.ts

@@ -2,6 +2,14 @@
 
 import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
 import { prisma } from "@/src/server/db";
+import { v4 as uuidv4 } from "uuid";
+
+interface TraceAPIResponse {
+  data: Array<{
+    id: string;
+    [key: string]: unknown;
+  }>;
+}
 
 describe("/api/public/traces API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
@@ -65,6 +73,7 @@ describe("/api/public/traces API Endpoint", () => {
     await makeAPICall("POST", "/api/public/traces", {
       id: "trace-id",
       metadata: { key: "value" },
+      timestamp: "2021-01-01T00:00:00.000Z",
       release: "1.0.0",
       version: "5.0.0",
       public: false,
@@ -84,6 +93,7 @@ describe("/api/public/traces API Endpoint", () => {
       version: "5.0.0",
       public: false,
       userId: "user-1",
+      timestamp: new Date("2021-01-01T00:00:00.000Z"),
     });
   });
 
@@ -106,34 +116,82 @@ describe("/api/public/traces API Endpoint", () => {
     });
 
     // multiple tags
-    const traces = await makeAPICall(
+    const traces = await makeAPICall<TraceAPIResponse>(
       "GET",
       "/api/public/traces?tags=tag-2&tags=tag-3",
     );
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
-    const traceIds = traces.body.data.map((t: { id: string }) => t.id);
+    const traceIds = traces.body.data.map((t) => t.id);
     // check for equality ok as ordered by timestamp
     expect(traceIds).toEqual(["trace-3", "trace-1"]);
 
     // single tag
-    const traces2 = await makeAPICall("GET", "/api/public/traces?tags=tag-1");
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
-    const traceIds2 = traces2.body.data.map((t: { id: string }) => t.id);
+    const traces2 = await makeAPICall<TraceAPIResponse>(
+      "GET",
+      "/api/public/traces?tags=tag-1",
+    );
+    const traceIds2 = traces2.body.data.map((t) => t.id);
     // check for equality ok as ordered by timestamp
     expect(traceIds2).toEqual(["trace-2", "trace-1"]);
 
     // wrong tag
-    const traces3 = await makeAPICall("GET", "/api/public/traces?tags=tag-10");
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
-    const traceIds3 = traces3.body.data.map((t: { id: string }) => t.id);
+    const traces3 = await makeAPICall<TraceAPIResponse>(
+      "GET",
+      "/api/public/traces?tags=tag-10",
+    );
+    const traceIds3 = traces3.body.data.map((t) => t.id);
     // check for equality ok as ordered by timestamp
     expect(traceIds3).toEqual([]);
 
     // no tag
-    const traces4 = await makeAPICall("GET", "/api/public/traces?tags=");
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
-    const traceIds4 = traces4.body.data.map((t: { id: string }) => t.id);
+    const traces4 = await makeAPICall<TraceAPIResponse>(
+      "GET",
+      "/api/public/traces?tags=",
+    );
+    const traceIds4 = traces4.body.data.map((t) => t.id);
     // check for equality ok as ordered by timestamp
     expect(traceIds4).toEqual(["trace-3", "trace-2", "trace-1"]);
   });
+
+  it("should handle totalCost and latency correctly", async () => {
+    await pruneDatabase();
+
+    // Create a trace with some observations that have costs and latencies
+    const traceId = uuidv4();
+    await makeAPICall("POST", "/api/public/traces", {
+      id: traceId,
+      name: "trace-with-costs",
+      userId: "user-costs",
+      projectId: "project-costs",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "2.0.0",
+    });
+    console.log(traceId);
+
+    // Simulate observations with costs and latencies
+    await makeAPICall("POST", "/api/public/generations", {
+      traceId: traceId,
+      usage: { totalCost: 10.5 },
+      startTime: "2021-01-01T00:00:00.000Z",
+      endTime: "2021-01-01T00:10:00.000Z",
+    });
+    await makeAPICall("POST", "/api/public/generations", {
+      traceId: traceId,
+      usage: { totalCost: 5.25 },
+      startTime: "2021-01-01T00:10:00.000Z",
+      endTime: "2021-01-01T00:20:00.000Z",
+    });
+
+    // Retrieve the trace with totalCost and latency
+    const traces = await makeAPICall<TraceAPIResponse>(
+      "GET",
+      `/api/public/traces`,
+    );
+    const traceData = traces.body.data[0];
+    if (!traceData) throw new Error("traceData is undefined");
+
+    // Check if the totalCost and latency are calculated correctly
+    expect(traceData.totalCost).toBeCloseTo(15.75); // Sum of costs
+    expect(traceData.latency).toBeCloseTo(1200); // Difference in seconds between min startTime and max endTime
+  });
 });

src/__tests__/usage.servertest.ts

@@ -1,3 +1,4 @@
+import { ModelUsageUnit } from "@/src/constants";
 import { tokenCount } from "@/src/features/ingest/lib/usage";
 
 describe("Token Count Functions", () => {
@@ -19,7 +20,7 @@ describe("Token Count Functions", () => {
       inputPrice: null,
       outputPrice: null,
       totalPrice: null,
-      unit: "TOKENS",
+      unit: ModelUsageUnit.Tokens,
     };
   };
 

src/components/LangfuseLogo.tsx

@@ -1,7 +1,8 @@
-import { VERSION } from "@/src/constants/VERSION";
+import { AlertTriangle } from "lucide-react";
+
+import { VERSION } from "@/src/constants";
 import { env } from "@/src/env.mjs";
 import { cn } from "@/src/utils/tailwind";
-import { AlertTriangle } from "lucide-react";
 
 export const LangfuseIcon = ({
   size = 32,

src/components/date-picker.tsx

@@ -31,7 +31,7 @@ import { type DashboardDateRange } from "@/src/pages/project/[projectId]";
 import { isValidOption } from "@/src/utils/types";
 import { setBeginningOfDay, setEndOfDay } from "@/src/utils/dates";
 
-export const DEFAULT_DATE_RANGE_SELECTION = "Select date range" as const;
+export const DEFAULT_DATE_RANGE_SELECTION = "Date range" as const;
 export type AvailableDateRangeSelections =
   | typeof DEFAULT_DATE_RANGE_SELECTION
   | DateTimeAggregationOption;
@@ -163,7 +163,7 @@ export function DatePickerWithRange({
             id="date"
             variant={"outline"}
             className={cn(
-              "w-[350px] justify-start text-left font-normal",
+              "w-[330px] justify-start text-left font-normal",
               !internalDateRange && "text-muted-foreground",
             )}
           >
@@ -171,8 +171,8 @@ export function DatePickerWithRange({
             {internalDateRange?.from ? (
               internalDateRange.to ? (
                 <>
-                  {format(internalDateRange.from, "LLL dd, y : hh:mm")} -{" "}
-                  {format(internalDateRange.to, "LLL dd, y : hh:mm")}
+                  {format(internalDateRange.from, "LLL dd, yy : hh:mm")} -{" "}
+                  {format(internalDateRange.to, "LLL dd, yy : hh:mm")}
                 </>
               ) : (
                 format(internalDateRange.from, "LLL dd, y")
@@ -194,7 +194,7 @@ export function DatePickerWithRange({
         </PopoverContent>
       </Popover>
       <Select value={selectedOption} onValueChange={onDropDownSelection}>
-        <SelectTrigger className="w-40 hover:bg-accent hover:text-accent-foreground focus:ring-0 focus:ring-offset-0">
+        <SelectTrigger className="w-[120px]  hover:bg-accent hover:text-accent-foreground focus:ring-0 focus:ring-offset-0">
           <SelectValue placeholder="Select" />
         </SelectTrigger>
         <SelectContent position="popper" defaultValue={60}>

src/components/delete-trace.tsx

@@ -30,12 +30,9 @@ export function DeleteTrace({
   const mutDeleteTraces = api.traces.deleteMany.useMutation({
     onSuccess: () => {
       setIsDeleted(true);
+      void utils.traces.invalidate();
       if (!isTableAction) {
-        void router
-          .push(`/project/${projectId}/traces`)
-          .then(() => utils.traces.invalidate());
-      } else {
-        void utils.traces.invalidate();
+        void router.push(`/project/${projectId}/traces`);
       }
     },
   });

src/components/layouts/header.tsx

@@ -40,7 +40,7 @@ export default function Header({
     [...props.breadcrumb.map((i) => i.href).filter(Boolean)].pop();
 
   return (
-    <div className={cn(level === "h2" ? "mb-8" : "mb-1")}>
+    <div className={cn(level === "h2" ? "mb-4" : "mb-1")}>
       <div>
         {backHref ? (
           <nav className="sm:hidden" aria-label="Back">

src/components/layouts/layout.tsx

@@ -8,7 +8,6 @@ import { useRouter } from "next/router";
 import clsx from "clsx";
 import { Code, MessageSquarePlus, Info, ChevronRightIcon } from "lucide-react";
 import { signOut, useSession } from "next-auth/react";
-import { ChevronDownIcon } from "@heroicons/react/20/solid";
 import { cn } from "@/src/utils/tailwind";
 import {
   Avatar,
@@ -24,6 +23,13 @@ import { env } from "@/src/env.mjs";
 import { LangfuseLogo } from "@/src/components/LangfuseLogo";
 import { Spinner } from "@/src/components/layouts/spinner";
 import { hasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { Toaster } from "@/src/components/ui/sonner";
+import {
+  NOTIFICATIONS,
+  useCheckNotification,
+} from "@/src/features/notifications/checkNotifications";
+import { ChevronDownIcon } from "@heroicons/react/20/solid";
+import useLocalStorage from "@/src/components/useLocalStorage";
 
 const userNavigation = [
   {
@@ -46,6 +52,9 @@ export default function Layout(props: PropsWithChildren) {
   const [sidebarOpen, setSidebarOpen] = useState(false);
   const router = useRouter();
   const session = useSession();
+
+  useCheckNotification(NOTIFICATIONS, session.status === "authenticated");
+
   const enableExperimentalFeatures =
     api.environment.enableExperimentalFeatures.useQuery().data ?? false;
 
@@ -68,7 +77,12 @@ export default function Layout(props: PropsWithChildren) {
     // RBAC
     if (
       route.rbacScope !== undefined &&
-      (!projectId || !hasAccess({ projectId, scope: route.rbacScope, session }))
+      (!projectId ||
+        !hasAccess({
+          projectId,
+          scope: route.rbacScope,
+          session: session.data,
+        }))
     )
       return null;
 
@@ -483,26 +497,27 @@ export default function Layout(props: PropsWithChildren) {
                   <Info className="h-4 w-4" />
                   <span className="font-semibold">DEMO (view-only)</span>
                 </div>
-                <div>Live data from the Langfuse Q&A Chatbot.</div>
+                <div>Use demo RAG chat to see live data in this project.</div>
               </div>
 
               <Button size="sm" asChild className="ml-2">
                 <Link
                   href={
                     env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "EU"
-                      ? "https://langfuse.com/docs/qa-chatbot"
-                      : "https://docs-staging.langfuse.com/docs/qa-chatbot"
+                      ? "https://langfuse.com/docs/demo"
+                      : "https://docs-staging.langfuse.com/docs/demo"
                   }
                   target="_blank"
                 >
                   {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "EU"
-                    ? "Q&A Chatbot ↗"
-                    : "Q&A Chatbot (staging) ↗"}
+                    ? "Use Chat ↗"
+                    : "Use Chat (staging) ↗"}
                 </Link>
               </Button>
             </div>
           ) : null}
           <main className="p-4">{props.children}</main>
+          <Toaster visibleToasts={1} />
         </div>
       </div>
     </>
@@ -522,29 +537,11 @@ const MainNavigation: React.FC<{
   nav: NavigationItem[];
   onNavitemClick?: () => void;
 }> = ({ nav, onNavitemClick }) => {
-  const STORAGE_KEY = "sidebar-tracing-default-open";
-  const getDefaultOpen = () => {
-    const savedState = localStorage.getItem(STORAGE_KEY);
-    if (savedState !== null) {
-      try {
-        return JSON.parse(savedState) as boolean;
-      } catch (e) {
-        console.error("Error parsing saved state: ", e);
-      }
-    }
-    return false;
-  };
+  const [isOpen, setIsOpen] = useLocalStorage(
+    "sidebar-tracing-default-open",
+    false,
+  );
 
-  const handleDropDownClick = () => {
-    const savedState = localStorage.getItem(STORAGE_KEY);
-    const isOpen =
-      savedState !== null ? (JSON.parse(savedState) as boolean) : false;
-    const newState = !isOpen;
-    localStorage.setItem(
-      "sidebar-tracing-default-open",
-      JSON.stringify(newState),
-    );
-  };
   return (
     <li>
       <ul role="list" className="-mx-2 space-y-1">
@@ -590,15 +587,14 @@ const MainNavigation: React.FC<{
               <Disclosure
                 as="div"
                 defaultOpen={
-                  item.children.some((child) => child.current) ||
-                  getDefaultOpen()
+                  item.children.some((child) => child.current) || isOpen
                 }
               >
                 {({ open }) => (
                   <>
                     <Disclosure.Button
                       className="group flex w-full items-center gap-x-3 rounded-md p-2 text-left text-sm font-semibold leading-6 hover:bg-gray-50 hover:text-indigo-600"
-                      onClick={handleDropDownClick}
+                      onClick={() => setIsOpen(!isOpen)}
                     >
                       {item.icon && (
                         <item.icon
@@ -656,7 +652,12 @@ const MainNavigation: React.FC<{
             ) : null}
           </li>
         ))}
-        <FeedbackButtonWrapper className="w-full">
+        <FeedbackButtonWrapper
+          className="w-full"
+          title="Provide feedback"
+          description="What do you think about this project? What can be improved?"
+          type="feedback"
+        >
           <li className="group flex cursor-pointer gap-x-3 rounded-md p-2 text-sm font-semibold leading-6 text-gray-700 hover:bg-gray-50 hover:text-indigo-600">
             <MessageSquarePlus
               className="h-6 w-6 shrink-0 text-gray-400 group-hover:text-indigo-600"

src/components/session/index.tsx

@@ -10,6 +10,7 @@ import { ManualScoreButton } from "@/src/features/manual-scoring/components/Manu
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { api } from "@/src/utils/api";
+import { usdFormatter } from "@/src/utils/numbers";
 import Link from "next/link";
 import { useEffect } from "react";
 
@@ -88,6 +89,11 @@ export const SessionPage: React.FC<{
           </Link>
         ))}
         <Badge variant="outline">Traces: {session.data?.traces.length}</Badge>
+        {session.data && (
+          <Badge variant="outline">
+            Total cost: {usdFormatter(session.data.totalCost, 2, 2)}
+          </Badge>
+        )}
       </div>
       <div className="mt-5 flex flex-col gap-2 border-t pt-5">
         {session.data?.traces.map((trace) => (

src/components/table/table-link.tsx

@@ -17,7 +17,7 @@ export default function TableLink({
       href={path}
       title={value}
     >
-      {value.length > truncateAt
+      {value.length - truncateAt > 3
         ? `...${value.substring(value.length - truncateAt)}`
         : value}
     </Link>

src/components/table/use-cases/models.tsx

@@ -31,11 +31,11 @@ const modelConfigDescriptions = {
     "Regex pattern to match `model` parameter of generations to model pricing",
   startDate:
     "Date to start pricing model. If not set, model is active unless a more recent version exists.",
-  inputPrice: "Price per unit of input",
-  outputPrice: "Price per unit of output",
+  inputPrice: "Price per 1000 units of input",
+  outputPrice: "Price per 1000 units of output",
   totalPrice:
-    "Price per unit, for models that don't have input/output specific prices",
-  unit: "Unit of measurement for model, can be TOKENS or CHARACTERS.",
+    "Price per 1000 units, for models that don't have input/output specific prices",
+  unit: "Unit of measurement for generative model, can be TOKENS, CHARACTERS, SECONDS, MILLISECONDS, or IMAGES.",
   tokenizerId:
     "Tokenizer used for this model to calculate token counts if none are ingested. Pick from list of supported tokenizers.",
   config:
@@ -107,7 +107,14 @@ export default function ModelTable({ projectId }: { projectId: string }) {
     {
       accessorKey: "inputPrice",
       id: "inputPrice",
-      header: "Input Price",
+      header: () => {
+        return (
+          <>
+            Input Price{" "}
+            <span className="text-xs text-gray-400">/ 1k units</span>
+          </>
+        );
+      },
       headerTooltip: {
         description: modelConfigDescriptions.inputPrice,
       },
@@ -116,7 +123,7 @@ export default function ModelTable({ projectId }: { projectId: string }) {
 
         return value ? (
           <span className="text-xs">
-            {usdFormatter(value.toNumber(), 2, 8)}
+            {usdFormatter(value.toNumber() * 1000, 2, 8)}
           </span>
         ) : (
           <span className="text-xs">-</span>
@@ -129,13 +136,20 @@ export default function ModelTable({ projectId }: { projectId: string }) {
       headerTooltip: {
         description: modelConfigDescriptions.outputPrice,
       },
-      header: "Output Price",
+      header: () => {
+        return (
+          <>
+            Output Price{" "}
+            <span className="text-xs text-gray-400">/ 1k units</span>
+          </>
+        );
+      },
       cell: ({ row }) => {
         const value: Decimal | undefined = row.getValue("outputPrice");
 
         return value ? (
           <span className="text-xs">
-            {usdFormatter(value.toNumber(), 2, 8)}
+            {usdFormatter(value.toNumber() * 1000, 2, 8)}
           </span>
         ) : (
           <span className="text-xs">-</span>
@@ -145,7 +159,14 @@ export default function ModelTable({ projectId }: { projectId: string }) {
     {
       accessorKey: "totalPrice",
       id: "totalPrice",
-      header: "Total Price",
+      header: () => {
+        return (
+          <>
+            Total Price{" "}
+            <span className="text-xs text-gray-400">/ 1k units</span>
+          </>
+        );
+      },
       headerTooltip: {
         description: modelConfigDescriptions.totalPrice,
       },
@@ -154,7 +175,7 @@ export default function ModelTable({ projectId }: { projectId: string }) {
 
         return value ? (
           <span className="text-xs">
-            {usdFormatter(value.toNumber(), 2, 8)}
+            {usdFormatter(value.toNumber() * 1000, 2, 8)}
           </span>
         ) : (
           <span className="text-xs">-</span>

src/components/table/use-cases/scores.tsx

@@ -4,6 +4,7 @@ import TableLink from "@/src/components/table/table-link";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
 import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
 import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
+import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { scoresTableColsWithOptions } from "@/src/server/api/definitions/scoresTable";
 import { api } from "@/src/utils/api";
 import { type RouterInput } from "@/src/utils/types";
@@ -49,11 +50,17 @@ export default function ScoresTable({
       ])
     : userFilterState;
 
+  const [orderByState, setOrderByState] = useOrderByState({
+    column: "timestamp",
+    order: "DESC",
+  });
+
   const scores = api.scores.all.useQuery({
     page: paginationState.pageIndex,
     limit: paginationState.pageSize,
     projectId,
     filter: filterState,
+    orderBy: orderByState,
   });
   const totalCount = scores.data?.slice(1)[0]?.totalCount ?? 0;
 
@@ -162,6 +169,8 @@ export default function ScoresTable({
           onChange: setPaginationState,
           state: paginationState,
         }}
+        orderBy={orderByState}
+        setOrderBy={setOrderByState}
         columnVisibility={columnVisibility}
         onColumnVisibilityChange={setColumnVisibility}
       />

src/components/table/use-cases/sessions.tsx

@@ -11,6 +11,7 @@ import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
 import { sessionsViewCols } from "@/src/server/api/definitions/sessionsView";
 import { api } from "@/src/utils/api";
 import { formatInterval, utcDateOffsetByDays } from "@/src/utils/dates";
+import { usdFormatter } from "@/src/utils/numbers";
 import { type RouterOutput } from "@/src/utils/types";
 import { useEffect } from "react";
 import { NumberParam, useQueryParams, withDefault } from "use-query-params";
@@ -22,6 +23,7 @@ export type SessionTableRow = {
   countTraces: number;
   bookmarked: boolean;
   sessionDuration: number | null;
+  totalCost: number;
 };
 
 export type SessionTableProps = {
@@ -98,6 +100,7 @@ export default function SessionsTable({
       countTraces: session.countTraces,
       bookmarked: session.bookmarked,
       sessionDuration: session.sessionDuration,
+      totalCost: session.totalCost,
     };
   };
 
@@ -132,6 +135,7 @@ export default function SessionsTable({
           <TableLink
             path={`/project/${projectId}/sessions/${encodeURIComponent(value)}`}
             value={value}
+            truncateAt={40}
           />
         ) : undefined;
       },
@@ -185,6 +189,20 @@ export default function SessionsTable({
       enableHiding: true,
       enableSorting: true,
     },
+    {
+      accessorKey: "totalCost",
+      id: "totalCost",
+      header: "Total Cost",
+      enableHiding: true,
+      enableSorting: true,
+      cell: ({ row }) => {
+        const value: number | undefined = row.getValue("totalCost");
+
+        return value !== undefined ? (
+          <span>{usdFormatter(value, 2, 2)}</span>
+        ) : undefined;
+      },
+    },
   ];
 
   const [columnVisibility, setColumnVisibility] =

src/components/table/use-cases/traces.tsx

@@ -185,6 +185,7 @@ export default function TracesTable({
             }
           }}
           aria-label="Select all"
+          className="opacity-60"
         />
       ),
       cell: ({ row }) => (
@@ -192,6 +193,7 @@ export default function TracesTable({
           checked={row.getIsSelected()}
           onCheckedChange={(value) => row.toggleSelected(!!value)}
           aria-label="Select row"
+          className="opacity-60"
         />
       ),
     },
@@ -439,9 +441,8 @@ export default function TracesTable({
         actionButtons={
           <TraceTableMultiSelectAction
             // Exclude traces that are not in the current page
-            selectedTraceIds={Object.keys(selectedRows).filter(
-              (traceId) =>
-                traces.data?.traces.map((t) => t.id).includes(traceId),
+            selectedTraceIds={Object.keys(selectedRows).filter((traceId) =>
+              traces.data?.traces.map((t) => t.id).includes(traceId),
             )}
             projectId={projectId}
             onDeleteSuccess={() => {

src/components/trace/IOPreview.tsx

@@ -35,11 +35,27 @@ export const IOPreview: React.FC<{
   if (!inOpenAiMessageArray.success) {
     // check if input is an array of length 1 including an array of OpenAiMessageSchema
     // this is the case for some integrations
+    // e.g. [[OpenAiMessageSchema, ...]]
     const inputArray = z.array(OpenAiMessageArraySchema).safeParse(input);
     if (inputArray.success && inputArray.data.length === 1) {
       inOpenAiMessageArray = OpenAiMessageArraySchema.safeParse(
         inputArray.data[0],
       );
+    } else {
+      // check if input is an object with a messages key
+      // this is the case for some integrations
+      // e.g. { messages: [OpenAiMessageSchema, ...] }
+      const inputObject = z
+        .object({
+          messages: OpenAiMessageArraySchema,
+        })
+        .safeParse(input);
+
+      if (inputObject.success) {
+        inOpenAiMessageArray = OpenAiMessageArraySchema.safeParse(
+          inputObject.data.messages,
+        );
+      }
     }
   }
   const outOpenAiMessage = OpenAiMessageSchema.safeParse(output);
@@ -102,9 +118,15 @@ export const IOPreview: React.FC<{
 
 const OpenAiMessageSchema = z
   .object({
-    role: z.enum(["system", "user", "assistant"]).optional(),
+    role: z.enum(["system", "user", "assistant", "function"]).optional(),
     name: z.string().optional(),
-    content: z.union([z.record(z.any()), z.string()]).nullable(),
+    content: z.union([z.record(z.any()).array(), z.string()]).nullable(),
+    function_call: z
+      .object({
+        name: z.string(),
+        arguments: z.record(z.any()),
+      })
+      .optional(),
   })
   .strict() // no additional properties
   .refine((value) => value.content !== null || value.role !== undefined);
@@ -131,7 +153,7 @@ const OpenAiMessageView: React.FC<{
           <Fragment key={index}>
             <JSONView
               title={message.name ?? message.role}
-              json={message.content}
+              json={message.function_call ?? message.content}
               className={cn(
                 message.role === "system" && "bg-gray-100",
                 message.role === "assistant" && "bg-green-50",

src/components/trace/ObservationTree.tsx

@@ -1,14 +1,21 @@
 import { type NestedObservation } from "@/src/utils/types";
 import { cn } from "@/src/utils/tailwind";
-import { type Trace, type Score } from "@prisma/client";
+import { type Trace, type Score, $Enums } from "@prisma/client";
 import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
 import { Fragment } from "react";
 import { type ObservationReturnType } from "@/src/server/api/routers/traces";
 import { LevelColors } from "@/src/components/level-colors";
 import { formatInterval } from "@/src/utils/dates";
+import { MinusCircle, MinusIcon, PlusCircleIcon, PlusIcon } from "lucide-react";
+import { Toggle } from "@/src/components/ui/toggle";
+import { Button } from "@/src/components/ui/button";
 
 export const ObservationTree = (props: {
   observations: ObservationReturnType[];
+  collapsedObservations: string[];
+  toggleCollapsedObservation: (id: string) => void;
+  collapseAll: () => void;
+  expandAll: () => void;
   trace: Trace;
   scores: Score[];
   currentObservationId: string | undefined;
@@ -21,6 +28,8 @@ export const ObservationTree = (props: {
   return (
     <div className={props.className}>
       <ObservationTreeTraceNode
+        expandAll={props.expandAll}
+        collapseAll={props.collapseAll}
         trace={props.trace}
         scores={props.scores}
         currentObservationId={props.currentObservationId}
@@ -30,6 +39,8 @@ export const ObservationTree = (props: {
       />
       <ObservationTreeNode
         observations={nestedObservations}
+        collapsedObservations={props.collapsedObservations}
+        toggleCollapsedObservation={props.toggleCollapsedObservation}
         scores={props.scores}
         indentationLevel={1}
         currentObservationId={props.currentObservationId}
@@ -40,8 +51,11 @@ export const ObservationTree = (props: {
     </div>
   );
 };
+
 const ObservationTreeTraceNode = (props: {
   trace: Trace & { latency?: number };
+  expandAll: () => void;
+  collapseAll: () => void;
   scores: Score[];
   currentObservationId: string | undefined;
   setCurrentObservationId: (id: string | undefined) => void;
@@ -50,7 +64,7 @@ const ObservationTreeTraceNode = (props: {
 }) => (
   <div
     className={cn(
-      "group mb-0.5 flex cursor-pointer flex-col gap-1 rounded-sm p-1.5",
+      "group mb-0.5 flex cursor-pointer flex-col gap-1 rounded-sm p-1",
       props.currentObservationId === undefined ||
         props.currentObservationId === ""
         ? "bg-gray-100"
@@ -60,7 +74,23 @@ const ObservationTreeTraceNode = (props: {
   >
     <div className="flex gap-2">
       <span className={cn("rounded-sm bg-gray-200 p-1 text-xs")}>TRACE</span>
-      <span className="text-sm">{props.trace.name}</span>
+      <span className="flex-1 text-sm">{props.trace.name}</span>
+      <Button
+        onClick={(ev) => (ev.stopPropagation(), props.expandAll())}
+        size="xs"
+        variant="ghost"
+        title="Expand all"
+      >
+        <PlusCircleIcon className="h-4 w-4" />
+      </Button>
+      <Button
+        onClick={(ev) => (ev.stopPropagation(), props.collapseAll())}
+        size="xs"
+        variant="ghost"
+        title="Collapse all"
+      >
+        <MinusCircle className="h-4 w-4" />
+      </Button>
     </div>
 
     {props.showMetrics && props.trace.latency ? (
@@ -79,8 +109,11 @@ const ObservationTreeTraceNode = (props: {
     ) : null}
   </div>
 );
+
 const ObservationTreeNode = (props: {
   observations: NestedObservation[];
+  collapsedObservations: string[];
+  toggleCollapsedObservation: (id: string) => void;
   scores: Score[];
   indentationLevel: number;
   currentObservationId: string | undefined;
@@ -91,96 +124,144 @@ const ObservationTreeNode = (props: {
   <>
     {props.observations
       .sort((a, b) => a.startTime.getTime() - b.startTime.getTime())
-      .map((observation) => (
-        <Fragment key={observation.id}>
-          <div className="flex">
-            {Array.from({ length: props.indentationLevel }, (_, i) => (
-              <div className="mx-2 border-r" key={i} />
-            ))}
-            <div
-              className={cn(
-                "group my-0.5 flex flex-1 cursor-pointer flex-col gap-1 rounded-sm p-1.5",
-                props.currentObservationId === observation.id
-                  ? "bg-gray-100"
-                  : "hover:bg-gray-50",
-              )}
-              onClick={() => props.setCurrentObservationId(observation.id)}
-            >
-              <div className="flex gap-2">
-                <span
-                  className={cn(
-                    "self-start rounded-sm bg-gray-200 p-1 text-xs",
-                  )}
-                >
-                  {observation.type}
-                </span>
-                <span className="line-clamp-1 text-sm">{observation.name}</span>
-              </div>
-              {props.showMetrics &&
-                (observation.promptTokens ||
-                  observation.completionTokens ||
-                  observation.totalTokens ||
-                  observation.endTime) && (
-                  <div className="flex gap-2">
-                    {observation.endTime ? (
-                      <span className="text-xs text-gray-500">
-                        {formatInterval(
-                          (observation.endTime.getTime() -
-                            observation.startTime.getTime()) /
-                            1000,
-                        )}
-                      </span>
-                    ) : null}
-                    {observation.promptTokens ||
-                    observation.completionTokens ||
-                    observation.totalTokens ? (
-                      <span className="text-xs text-gray-500">
-                        {observation.promptTokens} →{" "}
-                        {observation.completionTokens} (∑{" "}
-                        {observation.totalTokens})
-                      </span>
-                    ) : null}
-                  </div>
+      .map((observation) => {
+        const collapsed = props.collapsedObservations.includes(observation.id);
+
+        return (
+          <Fragment key={observation.id}>
+            <div className="flex">
+              {Array.from({ length: props.indentationLevel }, (_, i) => (
+                <div className="mx-2 border-r" key={i} />
+              ))}
+              <div
+                className={cn(
+                  "group my-0.5 flex flex-1 cursor-pointer flex-col gap-1 rounded-sm p-1",
+                  props.currentObservationId === observation.id
+                    ? "bg-gray-100"
+                    : "hover:bg-gray-50",
                 )}
-              {observation.level !== "DEFAULT" ? (
-                <div className="flex">
-                  <span
-                    className={cn(
-                      "rounded-sm p-0.5 text-xs",
-                      LevelColors[observation.level].bg,
-                      LevelColors[observation.level].text,
-                    )}
-                  >
-                    {observation.level}
-                  </span>
-                </div>
-              ) : null}
-              {props.showScores &&
-              props.scores.find((s) => s.observationId === observation.id) ? (
-                <div className="flex flex-wrap gap-1">
-                  <GroupedScoreBadges
-                    scores={props.scores.filter(
-                      (s) => s.observationId === observation.id,
-                    )}
+                onClick={() => props.setCurrentObservationId(observation.id)}
+              >
+                <div className="flex gap-2">
+                  <ColorCodedObservationType
+                    observationType={observation.type}
                   />
+                  <span className="line-clamp-1 flex-1 text-sm">
+                    {observation.name}
+                  </span>
+                  {observation.children.length === 0 ? null : (
+                    <Toggle
+                      onClick={(ev) => (
+                        ev.stopPropagation(),
+                        props.toggleCollapsedObservation(observation.id)
+                      )}
+                      variant="default"
+                      pressed={collapsed}
+                      size="xs"
+                      className="w-7"
+                      title={
+                        collapsed ? "Expand children" : "Collapse children"
+                      }
+                    >
+                      {collapsed ? (
+                        <PlusIcon className="h-4 w-4" />
+                      ) : (
+                        <MinusIcon className="h-4 w-4" />
+                      )}
+                    </Toggle>
+                  )}
                 </div>
-              ) : null}
+                {props.showMetrics &&
+                  (observation.promptTokens ||
+                    observation.completionTokens ||
+                    observation.totalTokens ||
+                    observation.endTime) && (
+                    <div className="flex gap-2">
+                      {observation.endTime ? (
+                        <span className="text-xs text-gray-500">
+                          {formatInterval(
+                            (observation.endTime.getTime() -
+                              observation.startTime.getTime()) /
+                              1000,
+                          )}
+                        </span>
+                      ) : null}
+                      {observation.promptTokens ||
+                      observation.completionTokens ||
+                      observation.totalTokens ? (
+                        <span className="text-xs text-gray-500">
+                          {observation.promptTokens} →{" "}
+                          {observation.completionTokens} (∑{" "}
+                          {observation.totalTokens})
+                        </span>
+                      ) : null}
+                    </div>
+                  )}
+                {observation.level !== "DEFAULT" ? (
+                  <div className="flex">
+                    <span
+                      className={cn(
+                        "rounded-sm p-0.5 text-xs",
+                        LevelColors[observation.level].bg,
+                        LevelColors[observation.level].text,
+                      )}
+                    >
+                      {observation.level}
+                    </span>
+                  </div>
+                ) : null}
+                {props.showScores &&
+                props.scores.find((s) => s.observationId === observation.id) ? (
+                  <div className="flex flex-wrap gap-1">
+                    <GroupedScoreBadges
+                      scores={props.scores.filter(
+                        (s) => s.observationId === observation.id,
+                      )}
+                    />
+                  </div>
+                ) : null}
+              </div>
             </div>
-          </div>
-          <ObservationTreeNode
-            observations={observation.children}
-            scores={props.scores}
-            indentationLevel={props.indentationLevel + 1}
-            currentObservationId={props.currentObservationId}
-            setCurrentObservationId={props.setCurrentObservationId}
-            showMetrics={props.showMetrics}
-            showScores={props.showScores}
-          />
-        </Fragment>
-      ))}
+            {!collapsed && (
+              <ObservationTreeNode
+                observations={observation.children}
+                collapsedObservations={props.collapsedObservations}
+                toggleCollapsedObservation={props.toggleCollapsedObservation}
+                scores={props.scores}
+                indentationLevel={props.indentationLevel + 1}
+                currentObservationId={props.currentObservationId}
+                setCurrentObservationId={props.setCurrentObservationId}
+                showMetrics={props.showMetrics}
+                showScores={props.showScores}
+              />
+            )}
+          </Fragment>
+        );
+      })}
   </>
 );
 
+const ColorCodedObservationType = (props: {
+  observationType: $Enums.ObservationType;
+}) => {
+  const colors: Record<$Enums.ObservationType, string> = {
+    [$Enums.ObservationType.SPAN]: "bg-blue-100",
+    [$Enums.ObservationType.GENERATION]: "bg-orange-100",
+    [$Enums.ObservationType.EVENT]: "bg-green-100",
+  };
+
+  return (
+    <span
+      className={cn(
+        "self-start rounded-sm p-1 text-xs",
+        colors[props.observationType],
+      )}
+    >
+      {props.observationType}
+    </span>
+  );
+};
+
 export function nestObservations(
   list: ObservationReturnType[],
 ): NestedObservation[] {

src/components/trace/index.tsx

@@ -20,9 +20,9 @@ import { TagTraceDetailsPopover } from "@/src/features/tag/components/TagTraceDe
 import useLocalStorage from "@/src/components/useLocalStorage";
 import { Toggle } from "@/src/components/ui/toggle";
 import { Award, ChevronsDownUp, ChevronsUpDown } from "lucide-react";
-import { ScrollArea } from "@/src/components/ui/scroll-area";
 import { usdFormatter } from "@/src/utils/numbers";
 import Decimal from "decimal.js";
+import { useCallback, useState } from "react";
 
 export function Trace(props: {
   observations: Array<ObservationReturnType>;
@@ -41,9 +41,57 @@ export function Trace(props: {
     true,
   );
 
+  const [collapsedObservations, setCollapsedObservations] = useState<string[]>(
+    [],
+  );
+
+  const toggleCollapsedObservation = useCallback(
+    (id: string) => {
+      if (collapsedObservations.includes(id)) {
+        setCollapsedObservations(collapsedObservations.filter((i) => i !== id));
+      } else {
+        setCollapsedObservations([...collapsedObservations, id]);
+      }
+    },
+    [collapsedObservations],
+  );
+
+  const collapseAll = useCallback(() => {
+    // exclude all parents of the current observation
+    let excludeParentObservations = new Set<string>();
+    let newExcludeParentObservations = new Set<string>();
+    do {
+      excludeParentObservations = new Set<string>([
+        ...excludeParentObservations,
+        ...newExcludeParentObservations,
+      ]);
+      newExcludeParentObservations = new Set<string>(
+        props.observations
+          .filter(
+            (o) =>
+              o.parentObservationId !== null &&
+              (o.id === currentObservationId ||
+                excludeParentObservations.has(o.id)),
+          )
+          .map((o) => o.parentObservationId as string)
+          .filter((id) => !excludeParentObservations.has(id)),
+      );
+    } while (newExcludeParentObservations.size > 0);
+
+    setCollapsedObservations(
+      props.observations
+        .map((o) => o.id)
+        .filter((id) => !excludeParentObservations.has(id)),
+    );
+  }, [props.observations, currentObservationId]);
+
+  const expandAll = useCallback(() => {
+    setCollapsedObservations([]);
+  }, [setCollapsedObservations]);
+
   return (
-    <div className="grid gap-4 md:h-full md:grid-cols-3">
-      <ScrollArea className="md:col-span-2 md:h-full">
+    <div className="grid gap-4 md:h-full md:grid-cols-5 lg:grid-cols-6 xl:grid-cols-7">
+      <div className="overflow-y-auto md:col-span-3 md:h-full lg:col-span-4 xl:col-span-5">
         {currentObservationId === undefined ||
         currentObservationId === "" ||
         currentObservationId === null ? (
@@ -61,8 +109,8 @@ export function Trace(props: {
             traceId={props.trace.id}
           />
         )}
-      </ScrollArea>
-      <div className="md:flex md:h-full md:flex-col md:overflow-hidden">
+      </div>
+      <div className="md:col-span-2 md:flex md:h-full md:flex-col md:overflow-hidden">
         <div className="mb-2 flex flex-shrink-0 flex-row justify-end gap-2">
           <Toggle
             pressed={scoresOnObservationTree}
@@ -89,17 +137,21 @@ export function Trace(props: {
             )}
           </Toggle>
         </div>
-        <ScrollArea className="flex flex-grow">
-          <ObservationTree
-            observations={props.observations}
-            trace={props.trace}
-            scores={props.scores}
-            currentObservationId={currentObservationId ?? undefined}
-            setCurrentObservationId={setCurrentObservationId}
-            showMetrics={metricsOnObservationTree}
-            showScores={scoresOnObservationTree}
-          />
-        </ScrollArea>
+
+        <ObservationTree
+          observations={props.observations}
+          collapsedObservations={collapsedObservations}
+          toggleCollapsedObservation={toggleCollapsedObservation}
+          collapseAll={collapseAll}
+          expandAll={expandAll}
+          trace={props.trace}
+          scores={props.scores}
+          currentObservationId={currentObservationId ?? undefined}
+          setCurrentObservationId={setCurrentObservationId}
+          showMetrics={metricsOnObservationTree}
+          showScores={scoresOnObservationTree}
+          className="flex w-full flex-col overflow-y-auto"
+        />
       </div>
     </div>
   );
@@ -139,7 +191,7 @@ export function TracePage({ traceId }: { traceId: string }) {
   if (trace.error?.data?.code === "UNAUTHORIZED") return <NoAccessError />;
   if (!trace.data) return <div>loading...</div>;
   return (
-    <div className="flex flex-col overflow-hidden xl:container md:h-[calc(100vh-2rem)]">
+    <div className="flex flex-col overflow-hidden 2xl:container md:h-[calc(100vh-2rem)]">
       <Header
         title="Trace Detail"
         breadcrumb={[

src/components/ui/code.tsx

@@ -16,13 +16,13 @@ export function JSONView(props: {
   const parsedJson = deepParseJson(props.json);
 
   return (
-    <div className={cn("max-w-full rounded-md border ", props.className)}>
+    <div className={cn("rounded-md border", props.className)}>
       {props.title ? (
         <div className="border-b px-3 py-1 text-xs font-medium">
           {props.title}
         </div>
       ) : undefined}
-      <div className="flex gap-2 whitespace-pre-wrap p-3 text-xs">
+      <div className="flex gap-2 whitespace-pre-wrap break-words p-3 text-xs">
         {props.isLoading ? (
           <Skeleton className="h-3 w-3/4" />
         ) : (
@@ -34,6 +34,7 @@ export function JSONView(props: {
             displaySize={"collapsed"}
             matchesURL={true}
             customizeCopy={(node) => stringifyJsonNode(node)}
+            className="w-full"
           />
         )}
       </div>

src/components/ui/sonner.tsx

@@ -0,0 +1,29 @@
+"use client";
+
+import { Toaster as Sonner } from "sonner";
+
+type ToasterProps = React.ComponentProps<typeof Sonner>;
+
+const Toaster = ({ ...props }: ToasterProps) => {
+  return (
+    <Sonner
+      theme={"light"}
+      className="toaster group"
+      position="top-right"
+      toastOptions={{
+        classNames: {
+          toast:
+            "group toast group-[.toaster]:bg-background group-[.toaster]:text-foreground group-[.toaster]:border-border group-[.toaster]:shadow-lg",
+          description: "group-[.toast]:text-muted-foreground",
+          actionButton:
+            "group-[.toast]:bg-primary group-[.toast]:text-primary-foreground",
+          cancelButton:
+            "group-[.toast]:bg-muted group-[.toast]:text-muted-foreground",
+        },
+      }}
+      {...props}
+    />
+  );
+};
+
+export { Toaster };

src/components/ui/toggle.tsx

@@ -17,6 +17,7 @@ const toggleVariants = cva(
       },
       size: {
         default: "h-10 px-3",
+        xs: "h-6 px-1.5",
         sm: "h-9 px-2.5",
         lg: "h-11 px-5",
       },

src/components/useLocalStorage.tsx

@@ -20,6 +20,9 @@ import { useState, useEffect } from "react";
  */
 function useLocalStorage<T>(localStorageKey: string, initialValue: T) {
   const [value, setValue] = useState<T>(() => {
+    if (typeof window === "undefined") {
+      return initialValue;
+    }
     try {
       const storedValue = localStorage.getItem(localStorageKey);
       return storedValue ? (JSON.parse(storedValue) as T) : initialValue;

src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.1.2";
+export const VERSION = "v2.6.0";

src/constants/index.ts

@@ -0,0 +1,9 @@
+export { VERSION } from "./VERSION";
+
+export enum ModelUsageUnit {
+  Characters = "CHARACTERS",
+  Tokens = "TOKENS",
+  Seconds = "SECONDS",
+  Milliseconds = "MILLISECONDS",
+  Images = "IMAGES",
+}

src/env.mjs

@@ -26,6 +26,7 @@ export const env = createEnv({
     ),
     NEXTAUTH_COOKIE_DOMAIN: z.string().optional(),
     LANGFUSE_TEAM_SLACK_WEBHOOK: z.string().url().optional(),
+    LANGFUSE_TEAM_BETTERSTACK_TOKEN: z.string().optional(),
     LANGFUSE_NEW_USER_SIGNUP_WEBHOOK: z.string().url().optional(),
     // Add `.min(1) on ID and SECRET if you want to make sure they're not empty
     LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: z.enum(["true", "false"]).optional(),
@@ -57,6 +58,8 @@ export const env = createEnv({
     S3_SECRET_ACCESS_KEY: z.string().optional(),
     S3_BUCKET_NAME: z.string().optional(),
     S3_REGION: z.string().optional(),
+    // Database exports
+    DB_EXPORT_PAGE_SIZE: z.number().optional(),
   },
 
   /**
@@ -91,6 +94,8 @@ export const env = createEnv({
     LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:
       process.env.LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES,
     LANGFUSE_TEAM_SLACK_WEBHOOK: process.env.LANGFUSE_TEAM_SLACK_WEBHOOK,
+    LANGFUSE_TEAM_BETTERSTACK_TOKEN:
+      process.env.LANGFUSE_TEAM_BETTERSTACK_TOKEN,
     LANGFUSE_NEW_USER_SIGNUP_WEBHOOK:
       process.env.LANGFUSE_NEW_USER_SIGNUP_WEBHOOK,
     SALT: process.env.SALT,
@@ -117,5 +122,10 @@ export const env = createEnv({
     S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY,
     S3_BUCKET_NAME: process.env.S3_BUCKET_NAME,
     S3_REGION: process.env.S3_REGION,
+    // Database exports
+    DB_EXPORT_PAGE_SIZE: process.env.DB_EXPORT_PAGE_SIZE,
   },
+  // Skip validation in Docker builds
+  // DOCKER_BUILD is set in Dockerfile
+  skipValidation: process.env.DOCKER_BUILD === "1",
 });

src/features/audit-logs/auditLog.ts

@@ -0,0 +1,57 @@
+import { prisma as _prisma } from "@/src/server/db";
+import { type MembershipRole } from "@prisma/client";
+
+export type AuditableResource =
+  | "membership"
+  | "membershipInvitation"
+  | "datasetItem"
+  | "dataset"
+  | "trace"
+  | "project"
+  | "observation"
+  | "score"
+  | "model"
+  | "prompt"
+  | "session"
+  | "apiKey";
+
+type AuditLog = {
+  resourceType: AuditableResource;
+  resourceId: string;
+  action: string;
+  before?: unknown;
+  after?: unknown;
+} & (
+  | {
+      projectId: string;
+      userId: string;
+      userProjectRole: MembershipRole;
+    }
+  | {
+      session: {
+        user: {
+          id: string;
+        };
+        projectRole: MembershipRole;
+        projectId: string;
+      };
+    }
+);
+
+export async function auditLog(log: AuditLog, prisma?: typeof _prisma) {
+  await (prisma ?? _prisma).auditLog.create({
+    data: {
+      projectId: "projectId" in log ? log.projectId : log.session.projectId,
+      userId: "userId" in log ? log.userId : log.session.user.id,
+      userProjectRole:
+        "userProjectRole" in log
+          ? log.userProjectRole
+          : log.session.projectRole,
+      resourceType: log.resourceType,
+      resourceId: log.resourceId,
+      action: log.action,
+      before: log.before ? JSON.stringify(log.before) : undefined,
+      after: log.after ? JSON.stringify(log.after) : undefined,
+    },
+  });
+}

src/features/betterstack/server/betterstack-webhook.ts

@@ -0,0 +1,28 @@
+import { env } from "@/src/env.mjs";
+
+export const sendToBetterstack = async (message: unknown) => {
+  if (!env.LANGFUSE_TEAM_BETTERSTACK_TOKEN)
+    throw new Error("LANGFUSE_TEAM_BETTERSTACK_TOKEN is not set");
+
+  const url = "https://in.logs.betterstack.com";
+
+  const headers = new Headers({
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${env.LANGFUSE_TEAM_BETTERSTACK_TOKEN}`,
+  });
+
+  const payload = {
+    dt: new Date().toISOString(), // Gets the current date in ISO format (UTC)
+    message: JSON.stringify(message, null, 2),
+  };
+
+  const response = await fetch(url, {
+    method: "POST",
+    headers: headers,
+    body: JSON.stringify(payload),
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP error! status: ${response.status}`);
+  }
+};

src/features/column-visibility/hooks/useColumnVisibility.ts

@@ -1,42 +1,28 @@
-import { useState, useEffect } from "react";
 import { type VisibilityState } from "@tanstack/react-table";
 import { type LangfuseColumnDef } from "@/src/components/table/types";
+import useLocalStorage from "@/src/components/useLocalStorage";
+import { useEffect } from "react";
 
 function useColumnVisibility<TData>(
   localStorageKey: string,
   columns: LangfuseColumnDef<TData>[],
 ) {
-  const [columnVisibility, setColumnVisibility] = useState<VisibilityState>(
-    () => {
-      try {
-        const savedVisibility = localStorage.getItem(localStorageKey);
-        const visibilityState: VisibilityState = savedVisibility
-          ? (JSON.parse(savedVisibility) as VisibilityState)
-          : {};
-        // set default visibility for columns that are not in the saved state
-        columns.forEach((column) => {
-          if (
-            "accessorKey" in column &&
-            typeof column.accessorKey === "string"
-          ) {
-            if (!(column.accessorKey in visibilityState)) {
-              visibilityState[column.accessorKey] =
-                column.defaultHidden === true ? false : true;
-            }
-          }
-        });
-        return visibilityState;
-      } catch (e) {
-        console.error("Error while loading saved column visibility", e);
-        return {};
+  const initialVisibilityState = () => {
+    const visibilityState: VisibilityState = {};
+    columns.forEach((column) => {
+      if ("accessorKey" in column && typeof column.accessorKey === "string") {
+        visibilityState[column.accessorKey] =
+          column.defaultHidden === true ? false : true;
       }
-    },
-  );
+    });
+    return visibilityState;
+  };
+
+  const [columnVisibility, setColumnVisibility] =
+    useLocalStorage<VisibilityState>(localStorageKey, initialVisibilityState());
 
   useEffect(() => {
-    const localStorageItem = localStorage.getItem(localStorageKey);
-
-    if (!localStorageItem || localStorageItem === "{}") {
+    if (Object.keys(columnVisibility).length === 0) {
       const initialVisibility: VisibilityState = {};
       columns.forEach((column) => {
         if ("accessorKey" in column && typeof column.accessorKey === "string") {
@@ -45,23 +31,8 @@ function useColumnVisibility<TData>(
         }
       });
       setColumnVisibility(initialVisibility);
-    } else {
-      // make sure all columns are in the visibility state
-      const visibilityState = JSON.parse(localStorageItem) as VisibilityState;
-      columns.forEach((column) => {
-        if ("accessorKey" in column && typeof column.accessorKey === "string") {
-          if (!(column.accessorKey in visibilityState)) {
-            visibilityState[column.accessorKey] =
-              column.defaultHidden === true ? false : true;
-          }
-        }
-      });
     }
-  }, [columns, localStorageKey]);
-
-  useEffect(() => {
-    localStorage.setItem(localStorageKey, JSON.stringify(columnVisibility));
-  }, [columnVisibility, localStorageKey]);
+  }, [columnVisibility, columns, setColumnVisibility]);
 
   return [columnVisibility, setColumnVisibility] as const;
 }

src/features/dashboard/components/BaseTimeSeriesChart.tsx

@@ -68,6 +68,8 @@ export function BaseTimeSeriesChart(props: {
       noDataText="No data"
       showLegend={props.showLegend}
       showAnimation={true}
+      onValueChange={() => {}}
+      enableLegendSlider={true}
     />
   );
 }

src/features/dashboard/components/ChartScores.tsx

@@ -13,6 +13,7 @@ import {
 } from "@/src/features/dashboard/components/hooks";
 import { NoData } from "@/src/features/dashboard/components/NoData";
 import DocPopup from "@/src/components/layouts/doc-popup";
+import { createTracesTimeFilter } from "@/src/features/dashboard/lib/dashboard-utils";
 
 export function ChartScores(props: {
   className?: string;
@@ -25,9 +26,7 @@ export function ChartScores(props: {
       projectId: props.projectId,
       from: "traces_scores",
       select: [{ column: "scoreName" }, { column: "value", agg: "AVG" }],
-      filter: props.globalFilterState.map((f) =>
-        f.type === "datetime" ? { ...f, column: "timestamp" } : f,
-      ),
+      filter: createTracesTimeFilter(props.globalFilterState),
       groupBy: [
         {
           type: "datetime",

src/features/dashboard/components/LatencyChart.tsx

@@ -30,9 +30,10 @@ export const LatencyChart = ({
   const latencies = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "observations",
+      from: "traces_observations",
       select: [
         { column: "duration", agg: "50thPercentile" },
+        { column: "duration", agg: "75thPercentile" },
         { column: "duration", agg: "90thPercentile" },
         { column: "duration", agg: "95thPercentile" },
         { column: "duration", agg: "99thPercentile" },
@@ -83,6 +84,10 @@ export const LatencyChart = ({
       tabTitle: "50th Percentile",
       data: getData("percentile50Duration"),
     },
+    {
+      tabTitle: "75th Percentile",
+      data: getData("percentile75Duration"),
+    },
     {
       tabTitle: "90th Percentile",
       data: getData("percentile90Duration"),

src/features/dashboard/components/MetricTable.tsx

@@ -19,7 +19,7 @@ export const MetricTable = ({
   const metrics = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "observations",
+      from: "traces_observations",
       select: [
         { column: "calculatedTotalCost", agg: "SUM" },
         { column: "totalTokens", agg: "SUM" },

src/features/dashboard/components/ModelUsageChart.tsx

@@ -33,7 +33,7 @@ export const ModelUsageChart = ({
   const tokens = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "observations",
+      from: "traces_observations",
       select: [
         { column: "totalTokens", agg: "SUM" },
         { column: "calculatedTotalCost", agg: "SUM" },

src/features/dashboard/components/ScoresTable.tsx

@@ -7,6 +7,7 @@ import { api } from "@/src/utils/api";
 import { compactNumberFormatter } from "@/src/utils/numbers";
 import { RightAlignedCell } from "./RightAlignedCell";
 import { TotalMetric } from "./TotalMetric";
+import { createTracesTimeFilter } from "@/src/features/dashboard/lib/dashboard-utils";
 
 export const ScoresTable = ({
   className,
@@ -17,11 +18,7 @@ export const ScoresTable = ({
   projectId: string;
   globalFilterState: FilterState;
 }) => {
-  const localFilters = globalFilterState.map((f) => ({
-    ...f,
-    column: "timestamp",
-  }));
-
+  const localFilters = createTracesTimeFilter(globalFilterState);
   const metrics = api.dashboard.chart.useQuery(
     {
       projectId,

src/features/dashboard/components/TabsComponent.tsx

@@ -21,6 +21,7 @@ export const TabComponent = ({ tabs }: TabComponentProps) => {
           name="tabs"
           className="block w-full rounded-md border-gray-300 py-2 pl-3 pr-10 text-base focus:border-indigo-500 focus:outline-none focus:ring-indigo-500 sm:text-sm"
           defaultValue={0}
+          onChange={(e) => setSelectedIndex(Number(e.target.selectedIndex))}
         >
           {tabs.map((tab) => (
             <option key={tab.tabTitle}>{tab.tabTitle}</option>

src/features/dashboard/components/UserChart.tsx

@@ -10,6 +10,7 @@ import { ExpandListButton } from "@/src/features/dashboard/components/cards/Chev
 import { useState } from "react";
 import DocPopup from "@/src/components/layouts/doc-popup";
 import { NoData } from "@/src/features/dashboard/components/NoData";
+import { createTracesTimeFilter } from "@/src/features/dashboard/lib/dashboard-utils";
 
 type BarChartDataPoint = {
   name: string;
@@ -61,10 +62,7 @@ export const UserChart = ({
       projectId,
       from: "traces",
       select: [{ column: "user" }, { column: "traceId", agg: "COUNT" }],
-      filter: globalFilterState.map((f) => ({
-        ...f,
-        column: "timestamp",
-      })),
+      filter: createTracesTimeFilter(globalFilterState),
       groupBy: [
         {
           type: "string",

src/features/dashboard/components/hooks.ts

@@ -10,7 +10,7 @@ export const getAllModels = (
   const allModels = api.dashboard.chart.useQuery(
     {
       projectId,
-      from: "observations",
+      from: "traces_observations",
       select: [{ column: "model" }],
       filter: [
         ...globalFilterState,

src/features/dashboard/lib/dashboard-utils.ts

@@ -0,0 +1,15 @@
+import { type FilterState } from "@/src/features/filters/types";
+
+// traces do not have a startTime or endTime column, so we need to map these to the timestamp column
+export const createTracesTimeFilter = (filters: FilterState) => {
+  return filters.map((f) => {
+    if (f.column === "startTime" || f.column === "endTime") {
+      return {
+        ...f,
+        column: "timestamp",
+      };
+    } else {
+      return f;
+    }
+  });
+};

src/features/dashboard/server/dashboard-router.ts

@@ -9,11 +9,19 @@ import {
   protectedProjectProcedure,
 } from "@/src/server/api/trpc";
 import { executeQuery } from "@/src/server/api/services/query-builder";
-import { sqlInterface } from "@/src/server/api/services/sqlInterface";
+import {
+  filterInterface,
+  sqlInterface,
+} from "@/src/server/api/services/sqlInterface";
 
 export const dashboardRouter = createTRPCRouter({
   chart: protectedProjectProcedure
-    .input(sqlInterface.extend({ projectId: z.string() }))
+    .input(
+      sqlInterface.extend({
+        projectId: z.string(),
+        filter: filterInterface.optional(),
+      }),
+    )
     .query(async ({ input, ctx }) => {
       return await executeQuery(ctx.prisma, input.projectId, input);
     }),

src/features/datasets/server/dataset-router.ts

@@ -6,6 +6,7 @@ import {
 } from "@/src/server/api/trpc";
 import { type DatasetRuns, Prisma, type Dataset } from "@prisma/client";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
+import { auditLog } from "@/src/features/audit-logs/auditLog";
 
 export const datasetRouter = createTRPCRouter({
   allDatasets: protectedProjectProcedure
@@ -203,7 +204,7 @@ export const datasetRouter = createTRPCRouter({
         projectId: input.projectId,
         scope: "datasets:CUD",
       });
-      return ctx.prisma.datasetItem.update({
+      const datasetItem = await ctx.prisma.datasetItem.update({
         where: {
           id: input.datasetItemId,
           datasetId: input.datasetId,
@@ -226,6 +227,15 @@ export const datasetRouter = createTRPCRouter({
           status: input.status,
         },
       });
+      await auditLog({
+        session: ctx.session,
+        resourceType: "datasetItem",
+        resourceId: input.datasetItemId,
+        projectId: input.projectId,
+        action: "update",
+        after: datasetItem,
+      });
+      return datasetItem;
     }),
   createDataset: protectedProjectProcedure
     .input(z.object({ projectId: z.string(), name: z.string() }))
@@ -235,12 +245,23 @@ export const datasetRouter = createTRPCRouter({
         projectId: input.projectId,
         scope: "datasets:CUD",
       });
-      return ctx.prisma.dataset.create({
+      const dataset = await ctx.prisma.dataset.create({
         data: {
           name: input.name,
           projectId: input.projectId,
         },
       });
+
+      await auditLog({
+        session: ctx.session,
+        resourceType: "dataset",
+        resourceId: dataset.id,
+        projectId: input.projectId,
+        action: "create",
+        after: dataset,
+      });
+
+      return dataset;
     }),
   deleteDataset: protectedProjectProcedure
     .input(z.object({ projectId: z.string(), datasetId: z.string() }))
@@ -250,12 +271,21 @@ export const datasetRouter = createTRPCRouter({
         projectId: input.projectId,
         scope: "datasets:CUD",
       });
-      return ctx.prisma.dataset.delete({
+      const deletedDataset = await ctx.prisma.dataset.delete({
         where: {
           id: input.datasetId,
           projectId: input.projectId,
         },
       });
+      await auditLog({
+        session: ctx.session,
+        resourceType: "dataset",
+        resourceId: deletedDataset.id,
+        projectId: input.projectId,
+        action: "delete",
+        before: deletedDataset,
+      });
+      return deletedDataset;
     }),
   createDatasetItem: protectedProjectProcedure
     .input(
@@ -283,7 +313,7 @@ export const datasetRouter = createTRPCRouter({
         throw new Error("Dataset not found");
       }
 
-      return ctx.prisma.datasetItem.create({
+      const datasetItem = await ctx.prisma.datasetItem.create({
         data: {
           input: JSON.parse(input.input) as Prisma.InputJsonObject,
           expectedOutput:
@@ -296,6 +326,15 @@ export const datasetRouter = createTRPCRouter({
           sourceObservationId: input.sourceObservationId,
         },
       });
+      await auditLog({
+        session: ctx.session,
+        resourceType: "datasetItem",
+        resourceId: datasetItem.id,
+        projectId: input.projectId,
+        action: "create",
+        after: datasetItem,
+      });
+      return datasetItem;
     }),
   runitemsByRunIdOrItemId: protectedProjectProcedure
     .input(

src/features/feedback/component/FeedbackButton.tsx

@@ -1,4 +1,3 @@
-import * as Sentry from "@sentry/nextjs";
 import { Button } from "@/src/components/ui/button";
 import {
   Dialog,
@@ -18,17 +17,28 @@ import {
   FormItem,
   FormMessage,
 } from "@/src/components/ui/form";
-import { type PropsWithChildren, useState } from "react";
+import { useState } from "react";
 import { useSession } from "next-auth/react";
 import { Textarea } from "@/src/components/ui/textarea";
 
+interface FeedbackDialogProps {
+  className?: string;
+  children: React.ReactNode;
+  title: string;
+  description: string;
+  type: "feedback" | "dashboard";
+}
 const formSchema = z.object({
   feedback: z.string().min(3, "Must have at least 3 characters"),
 });
 
-export function FeedbackButtonWrapper(
-  props: PropsWithChildren<{ className?: string }>,
-) {
+export function FeedbackButtonWrapper({
+  className,
+  children,
+  title,
+  description,
+  type,
+}: FeedbackDialogProps) {
   const [open, setOpen] = useState(false);
   const session = useSession();
 
@@ -40,16 +50,6 @@ export function FeedbackButtonWrapper(
   });
 
   async function onSubmit(values: z.infer<typeof formSchema>) {
-    // Add to sentry
-    if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
-      const eventId = Sentry.captureMessage(`User submitted feedback`);
-      Sentry.captureUserFeedback({
-        event_id: eventId,
-        email: session.data?.user?.email ?? "",
-        name: session.data?.user?.name ?? "",
-        comments: values.feedback,
-      });
-    }
     try {
       const res = await fetch("https://cloud.langfuse.com/api/feedback", {
         method: "POST",
@@ -57,6 +57,7 @@ export function FeedbackButtonWrapper(
           "Content-Type": "application/json",
         },
         body: JSON.stringify({
+          type,
           ...values,
           url: window.location.href,
           user: session.data?.user,
@@ -84,27 +85,25 @@ export function FeedbackButtonWrapper(
 
   return (
     <Dialog open={open} onOpenChange={setOpen}>
-      <DialogTrigger className={props.className} asChild>
-        {props.children}
+      <DialogTrigger className={className} asChild>
+        {children}
       </DialogTrigger>
       <DialogContent>
         <DialogHeader>
-          <DialogTitle className="mb-5">Provide feedback</DialogTitle>
+          <DialogTitle>{title}</DialogTitle>
         </DialogHeader>
         <Form {...form}>
           <form
             // eslint-disable-next-line @typescript-eslint/no-misused-promises
             onSubmit={form.handleSubmit(onSubmit)}
-            className="space-y-8"
+            className="space-y-4"
           >
             <FormField
               control={form.control}
               name="feedback"
               render={({ field }) => (
                 <FormItem>
-                  <FormDescription>
-                    What do you think about this project? What can be improved?
-                  </FormDescription>
+                  <FormDescription>{description}</FormDescription>
                   <FormControl>
                     <Textarea {...field} />
                   </FormControl>

src/features/feedback/server/feedbackHandler.ts

@@ -1,5 +1,5 @@
-import { env } from "@/src/env.mjs";
 import { runFeedbackCorsMiddleware } from "@/src/features/feedback/server/corsMiddleware";
+import { sendToSlack } from "@/src/features/slack/server/slack-webhook";
 import { type NextApiRequest, type NextApiResponse } from "next";
 
 // Collects feedack from users that do not use the cloud version of the app
@@ -10,17 +10,7 @@ export default async function feedbackApiHandler(
   await runFeedbackCorsMiddleware(req, res);
 
   try {
-    if (!env.LANGFUSE_TEAM_SLACK_WEBHOOK)
-      throw new Error("LANGFUSE_TEAM_SLACK_WEBHOOK is not set");
-
-    const slackResponse = await fetch(env.LANGFUSE_TEAM_SLACK_WEBHOOK, {
-      method: "POST",
-      body: JSON.stringify({ rawBody: JSON.stringify(req.body, null, 2) }),
-      headers: {
-        "Content-Type": "application/json",
-      },
-    });
-
+    const slackResponse = await sendToSlack(req.body);
     if (slackResponse.status === 200) {
       res.status(200).json({ status: "OK" });
     } else {

src/features/filters/components/filter-builder.tsx

@@ -97,7 +97,9 @@ export function FilterBuilder({
                         ? new Date(filter.value).toLocaleDateString()
                         : filter.type === "stringOptions" ||
                             filter.type === "arrayOptions"
-                          ? filter.value.join(", ")
+                          ? filter.value.length > 2
+                            ? `${filter.value.length} selected`
+                            : filter.value.join(", ")
                           : filter.type === "number" ||
                               filter.type === "numberObject"
                             ? filter.value
@@ -292,18 +294,17 @@ function FilterBuilderForm({
                   ) : filter.type === "number" ||
                     filter.type === "numberObject" ? (
                     <Input
-                      value={filter.value?.toString() ?? ""}
+                      value={filter.value ?? undefined}
                       type="number"
                       step="0.01"
+                      lang="en-US"
                       onChange={(e) =>
                         handleFilterChange(
                           {
                             ...filter,
-                            value:
-                              isNaN(Number(e.target.value)) ||
-                              e.target.value.endsWith(".")
-                                ? e.target.value
-                                : Number(e.target.value),
+                            value: isNaN(Number(e.target.value))
+                              ? e.target.value
+                              : Number(e.target.value),
                           },
                           i,
                         )

src/features/filters/server/filterToPrisma.ts

@@ -1,6 +1,9 @@
 import { type FilterState } from "@/src/features/filters/types";
 import { filterOperators } from "@/src/server/api/interfaces/filters";
-import { type ColumnDefinition } from "@/src/server/api/interfaces/tableDefinition";
+import {
+  type TableNames as TableName,
+  type ColumnDefinition,
+} from "@/src/server/api/interfaces/tableDefinition";
 import { Prisma } from "@prisma/client";
 
 const operatorReplacements = {
@@ -18,11 +21,24 @@ const arrayOperatorReplacements = {
   "none of": "&&",
 };
 
-export function filterToPrismaSql(
+export function tableColumnsToSqlFilterAndPrefix(
   filters: FilterState,
   tableColumns: ColumnDefinition[],
+  table: TableName,
 ): Prisma.Sql {
-  const statements = filters.map((filter) => {
+  const sql = tableColumnsToSqlFilter(filters, tableColumns, table);
+  if (sql === Prisma.empty) {
+    return Prisma.empty;
+  }
+  return Prisma.join([Prisma.raw("AND "), sql], "");
+}
+
+export function tableColumnsToSqlFilter(
+  filters: FilterState,
+  tableColumns: ColumnDefinition[],
+  table: TableName,
+): Prisma.Sql {
+  const internalFilters = filters.map((filter) => {
     // Get column definition to map column to internal name, e.g. "t.id"
     const col = tableColumns.find(
       (c) =>
@@ -33,8 +49,17 @@ export function filterToPrismaSql(
       console.error("Invalid filter column", filter.column);
       throw new Error("Invalid filter column: " + filter.column);
     }
-
     const colPrisma = Prisma.raw(col.internal);
+    return {
+      condition: filter,
+      internalColumn: colPrisma,
+      column: col,
+      table: table,
+    };
+  });
+
+  const statements = internalFilters.map((filterAndColumn) => {
+    const filter = filterAndColumn.condition;
     const operatorPrisma =
       filter.type === "arrayOptions"
         ? Prisma.raw(
@@ -58,7 +83,7 @@ export function filterToPrismaSql(
         break;
       case "number":
       case "numberObject":
-        valuePrisma = Prisma.sql`${filter.value}::DOUBLE PRECISION`;
+        valuePrisma = Prisma.sql`${filter.value.toString()}::DOUBLE PRECISION`;
         break;
       case "string":
       case "stringObject":
@@ -108,18 +133,27 @@ export function filterToPrismaSql(
         ? [Prisma.raw("NOT ("), Prisma.raw(")")]
         : [Prisma.empty, Prisma.empty];
 
-    return Prisma.sql`${funcPrisma1}${cast1}${colPrisma}${jsonKeyPrisma}${cast2} ${operatorPrisma} ${valuePrefix}${valuePrisma}${valueSuffix}${funcPrisma2}`;
+    return Prisma.sql`${funcPrisma1}${cast1}${filterAndColumn.internalColumn}${jsonKeyPrisma}${cast2} ${operatorPrisma} ${valuePrefix}${valuePrisma}${castValueToPostgresTypes(filterAndColumn.column, filterAndColumn.table)}${valueSuffix}${funcPrisma2}`;
   });
   if (statements.length === 0) {
     return Prisma.empty;
   }
 
-  return Prisma.join(
-    [Prisma.raw("AND "), Prisma.join(statements, " AND ")],
-    "",
-  );
+  return Prisma.join(statements, " AND ");
 }
 
+const castValueToPostgresTypes = (
+  column: ColumnDefinition,
+  table: TableName,
+) => {
+  return column.name === "type" &&
+    (table === "observations" ||
+      table === "traces_observations" ||
+      table === "traces_parent_observation_scores")
+    ? Prisma.sql`::"ObservationType"`
+    : Prisma.empty;
+};
+
 const dateOperators = filterOperators["datetime"];
 
 export const datetimeFilterToPrismaSql = (

src/features/models/components/NewModelForm.tsx

@@ -1,7 +1,12 @@
-import { Button } from "@/src/components/ui/button";
-import * as z from "zod";
-import { zodResolver } from "@hookform/resolvers/zod";
+import { usePostHog } from "posthog-js/react";
+import { useState } from "react";
 import { useForm } from "react-hook-form";
+import JsonView from "react18-json-view";
+import * as z from "zod";
+
+import { DatePicker } from "@/src/components/date-picker";
+import Header from "@/src/components/layouts/header";
+import { Button } from "@/src/components/ui/button";
 import {
   Form,
   FormControl,
@@ -11,6 +16,7 @@ import {
   FormLabel,
   FormMessage,
 } from "@/src/components/ui/form";
+import { Input } from "@/src/components/ui/input";
 import {
   Select,
   SelectContent,
@@ -18,14 +24,10 @@ import {
   SelectTrigger,
   SelectValue,
 } from "@/src/components/ui/select";
-import { api } from "@/src/utils/api";
-import { useState } from "react";
-import { usePostHog } from "posthog-js/react";
-import { Input } from "@/src/components/ui/input";
-import { DatePicker } from "@/src/components/date-picker";
-import Header from "@/src/components/layouts/header";
+import { ModelUsageUnit } from "@/src/constants";
 import { AutoComplete } from "@/src/features/prompts/components/auto-complete";
-import JsonView from "react18-json-view";
+import { api } from "@/src/utils/api";
+import { zodResolver } from "@hookform/resolvers/zod";
 
 const formSchema = z.object({
   modelName: z.string().min(1),
@@ -49,7 +51,7 @@ const formSchema = z.object({
       message: "Price needs to be numeric",
     })
     .optional(),
-  unit: z.enum(["TOKENS", "CHARACTERS"]),
+  unit: z.nativeEnum(ModelUsageUnit),
   tokenizerId: z.enum(["openai", "claude", "None"]),
   tokenizerConfig: z.string().refine(
     (value) => {
@@ -81,7 +83,7 @@ export const NewModelForm = (props: {
       inputPrice: "",
       outputPrice: "",
       totalPrice: "",
-      unit: "TOKENS",
+      unit: ModelUsageUnit.Tokens,
       tokenizerId: "None",
       tokenizerConfig: "{}",
     },
@@ -233,7 +235,7 @@ export const NewModelForm = (props: {
                   </SelectTrigger>
                 </FormControl>
                 <SelectContent>
-                  {["TOKENS", "CHARACTERS"].map((unit) => (
+                  {Object.values(ModelUsageUnit).map((unit) => (
                     <SelectItem value={unit} key={unit}>
                       {unit}
                     </SelectItem>

src/features/notifications/Notification.tsx

@@ -0,0 +1,47 @@
+import { X } from "lucide-react";
+
+export interface TNotification {
+  id: number;
+  releaseDate: Date;
+  message: string | JSX.Element;
+  description?: JSX.Element | string;
+}
+
+interface NotificationProps {
+  notification: TNotification;
+  setLastSeenId: (id: number) => void;
+  dismissToast: (t?: string | number | undefined) => void;
+  toast: string | number;
+}
+
+const Notification: React.FC<NotificationProps> = ({
+  notification,
+  setLastSeenId,
+  dismissToast,
+  toast,
+}) => (
+  <div className="flex justify-between">
+    <div className="flex min-w-[300px] flex-1 flex-col justify-center">
+      <div className="m-0 text-sm font-medium leading-tight text-gray-800">
+        {notification.message}
+      </div>
+      {notification.description && (
+        <div className="mt-2 flex-1 text-sm leading-tight text-gray-800">
+          {notification.description}
+        </div>
+      )}
+    </div>
+    <button
+      className="flex h-6 w-6 cursor-pointer items-start justify-end border-none bg-transparent p-0 text-gray-800 transition-colors duration-200"
+      onClick={() => {
+        setLastSeenId(notification.id);
+        dismissToast(toast);
+      }}
+      aria-label="Close"
+    >
+      <X size={14} />
+    </button>
+  </div>
+);
+
+export default Notification;

src/features/notifications/checkNotifications.tsx

@@ -0,0 +1,95 @@
+import { useEffect } from "react";
+import Link from "next/link";
+import { toast } from "sonner";
+import useLocalStorage from "@/src/components/useLocalStorage";
+import Notification, {
+  type TNotification,
+} from "@/src/features/notifications/Notification";
+import { Button } from "@/src/components/ui/button";
+import { env } from "@/src/env.mjs";
+
+export const NOTIFICATIONS: TNotification[] = [
+  {
+    id: 1,
+    releaseDate: new Date("2024-01-29"),
+    message: "New: Custom model prices",
+    description: (
+      <div>
+        <p>
+          Langfuse now supports any LLM model for usage and cost tracking. The
+          highlights:
+        </p>
+        <ul className="ms-4 mt-2 list-outside list-disc">
+          <li>Define your model definitions (price, usage).</li>
+          <li>Optionally, ingest cost via the API/SDK</li>
+          <li>Support for usage in tokens, seconds and characters</li>
+        </ul>
+        {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === undefined && (
+          <p className="mt-2">
+            Self-hosted: The upgrade to v2 includes a breaking change. After
+            upgrading, you must run a migration script to ensure accurate usage
+            and cost calculation for previously ingested traces. Please refer to
+            the post for details.
+          </p>
+        )}
+        <Button size="sm" variant="secondary" className="mt-3">
+          <Link href="https://langfuse.com/changelog/2024-01-29-custom-model-prices">
+            Changelog post
+          </Link>
+        </Button>
+      </div>
+    ),
+  },
+];
+
+export const useCheckNotification = (
+  notification: TNotification[],
+  authenticated: boolean,
+) => {
+  const [lastSeenId, setLastSeenId] = useLocalStorage<number>(
+    "lastSeenNotificationId",
+    0,
+  );
+  useEffect(() => {
+    if (!authenticated) {
+      return;
+    }
+
+    const timeoutId = setTimeout(() => {
+      notification
+        .reverse()
+        .filter(
+          (n) =>
+            // only show notifications that are less than 30 days old
+            (new Date().getTime() - n.releaseDate.getTime()) /
+              (1000 * 60 * 60 * 24) <=
+            30,
+        )
+        .forEach((n) => {
+          if (n.id > lastSeenId) {
+            toast.custom(
+              (t) => (
+                <Notification
+                  notification={n}
+                  setLastSeenId={setLastSeenId}
+                  dismissToast={toast.dismiss}
+                  toast={t}
+                />
+              ),
+              {
+                // needed to upsert toasts in case it is rendered multiple times
+                id: n.id.toString(),
+                duration: 600_000, // 10 minutes
+                style: {
+                  padding: "1rem",
+                  border: "1px solid #e2e8f0",
+                  borderRadius: "0.5rem",
+                },
+              },
+            );
+          }
+        });
+    }, 1500);
+    return () => clearTimeout(timeoutId);
+  }, [lastSeenId, notification, setLastSeenId, authenticated]);
+};

src/features/orderBy/server/orderByToPrisma.clienttest.ts

@@ -1,27 +1,8 @@
 import { orderByToPrismaSql } from "@/src/features/orderBy/server/orderByToPrisma";
 import { tracesTableCols } from "@/src/server/api/definitions/tracesTable";
-import { Prisma } from "@prisma/client";
 
 // The test for the orderByToPrisma function
 describe("orderByToPrisma (Convert orderBy to Prisma.sql)", () => {
-  test("orderByToPrisma returns default sql when orderBy=null", () => {
-    expect(orderByToPrismaSql(null, tracesTableCols)).toStrictEqual(
-      Prisma.sql`ORDER BY t.timestamp DESC`,
-    );
-  });
-
-  test("orderByToPrisma returns correct clause for orderBy column included in column defs", () => {
-    expect(
-      orderByToPrismaSql(
-        {
-          column: "latency",
-          order: "ASC",
-        },
-        tracesTableCols,
-      ),
-    ).toStrictEqual(Prisma.sql`ORDER BY tl.latency ASC`);
-  });
-
   test("orderByToPrisma throws error for orderBy column not included in column defs", () => {
     expect(() =>
       orderByToPrismaSql(
@@ -33,4 +14,16 @@ describe("orderByToPrisma (Convert orderBy to Prisma.sql)", () => {
       ),
     ).toThrow(/Invalid filter column: InvalidCol/);
   });
+
+  test("orderByToPrisma throws error for orderBy order that is not valid", () => {
+    expect(() =>
+      orderByToPrismaSql(
+        {
+          column: "latency",
+          order: "test" as "ASC" | "DESC",
+        },
+        tracesTableCols,
+      ),
+    ).toThrow(/Invalid order: test/);
+  });
 });

src/features/orderBy/server/orderByToPrisma.ts

@@ -1,6 +1,7 @@
 import { type OrderByState } from "@/src/features/orderBy/types";
 import { type ColumnDefinition } from "@/src/server/api/interfaces/tableDefinition";
 import { Prisma } from "@prisma/client";
+import { z } from "zod";
 
 /**
  * Convert orderBy to SQL ORDER BY clause
@@ -13,7 +14,7 @@ export function orderByToPrismaSql(
   tableColumns: ColumnDefinition[],
 ): Prisma.Sql {
   if (!orderBy) {
-    return Prisma.sql([`ORDER BY t.timestamp DESC`]);
+    return Prisma.sql`ORDER BY t.timestamp DESC NULLS LAST`;
   }
   // Get column definition to map column to internal name, e.g. "t.id"
   const col = tableColumns.find(
@@ -21,10 +22,22 @@ export function orderByToPrismaSql(
     // It's less error-prone & decouples data fetching from the human-readable UI labels
     (c) => c.name === orderBy.column || c.id === orderBy.column,
   );
+
   if (!col) {
     console.log("Invalid filter column", orderBy.column);
     throw new Error("Invalid filter column: " + orderBy.column);
   }
 
-  return Prisma.sql([`ORDER BY ${col.internal} ${orderBy.order}`]);
+  // Assert that orderBy.order is either "asc" or "desc"
+  const orderByOrder = z.enum(["ASC", "DESC"]);
+  const order = orderByOrder.safeParse(orderBy.order);
+  if (!order.success) {
+    console.log("Invalid order", orderBy.order);
+    throw new Error("Invalid order: " + orderBy.order);
+  }
+
+  // Both column and order are safe, can use raw SQL
+  return Prisma.raw(
+    `ORDER BY ${col.internal} ${order.data} ${orderBy.order === "DESC" ? "NULLS LAST" : "NULLS FIRST"}`,
+  );
 }

src/features/pricing-page/PricingPage.tsx

@@ -9,7 +9,7 @@ export function PricingPage(props: { className?: string }) {
       <div className={props.className}>
         <stripe-pricing-table
           client-reference-id={session.data?.user?.id}
-          pricing-table-id="prctbl_1OWhugAWilt2EAVVKHi8TEw7"
+          pricing-table-id="prctbl_1OjTUyAWilt2EAVVMCMAMDgB"
           publishable-key="pk_live_51MPW00AWilt2EAVVFWfPTQhgmLA0EeacLSzAs6e3vECCcMBvwcMse81XgXO6k1bdBHbPBdpOmrXE8P1gBrxE7yhH00RPHQ8SyG"
         ></stripe-pricing-table>
       </div>

src/features/projects/server/projectsRouter.ts

@@ -1,8 +1,13 @@
-import { createTRPCRouter, protectedProcedure } from "@/src/server/api/trpc";
+import {
+  createTRPCRouter,
+  protectedProcedure,
+  protectedProjectProcedure,
+} from "@/src/server/api/trpc";
 import * as z from "zod";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { TRPCError } from "@trpc/server";
 import { projectNameSchema } from "@/src/features/auth/lib/projectNameSchema";
+import { auditLog } from "@/src/features/audit-logs/auditLog";
 
 export const projectsRouter = createTRPCRouter({
   all: protectedProcedure.query(async ({ ctx }) => {
@@ -66,6 +71,15 @@ export const projectsRouter = createTRPCRouter({
           },
         },
       });
+      await auditLog({
+        resourceType: "project",
+        resourceId: project.id,
+        action: "create",
+        userId: ctx.session.user.id,
+        projectId: project.id,
+        userProjectRole: "OWNER",
+        after: project,
+      });
 
       return {
         id: project.id,
@@ -74,7 +88,7 @@ export const projectsRouter = createTRPCRouter({
       };
     }),
 
-  update: protectedProcedure
+  update: protectedProjectProcedure
     .input(
       z.object({
         projectId: z.string(),
@@ -88,7 +102,7 @@ export const projectsRouter = createTRPCRouter({
         scope: "project:update",
       });
 
-      await ctx.prisma.project.update({
+      const project = await ctx.prisma.project.update({
         where: {
           id: input.projectId,
         },
@@ -96,10 +110,17 @@ export const projectsRouter = createTRPCRouter({
           name: input.newName,
         },
       });
+      await auditLog({
+        session: ctx.session,
+        resourceType: "project",
+        resourceId: input.projectId,
+        action: "update",
+        after: project,
+      });
       return true;
     }),
 
-  delete: protectedProcedure
+  delete: protectedProjectProcedure
     .input(
       z.object({
         projectId: z.string(),
@@ -111,6 +132,12 @@ export const projectsRouter = createTRPCRouter({
         projectId: input.projectId,
         scope: "project:delete",
       });
+      await auditLog({
+        session: ctx.session,
+        resourceType: "project",
+        resourceId: input.projectId,
+        action: "delete",
+      });
 
       await ctx.prisma.project.delete({
         where: {
@@ -121,7 +148,7 @@ export const projectsRouter = createTRPCRouter({
       return true;
     }),
 
-  transfer: protectedProcedure
+  transfer: protectedProjectProcedure
     .input(
       z.object({
         projectId: z.string(),
@@ -145,6 +172,14 @@ export const projectsRouter = createTRPCRouter({
       if (newOwner.id === ctx.session.user.id)
         throw new Error("You cannot transfer project to yourself");
 
+      await auditLog({
+        session: ctx.session,
+        resourceType: "project",
+        resourceId: input.projectId,
+        action: "transfer",
+        after: { ownerId: newOwner.id },
+      });
+
       return ctx.prisma.$transaction([
         // Add new owner, upsert to update role if already exists
         ctx.prisma.membership.upsert({

src/features/prompts/components/delete-prompt-version.tsx

@@ -0,0 +1,90 @@
+import { Button } from "@/src/components/ui/button";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { api } from "@/src/utils/api";
+import { Trash2 } from "lucide-react";
+import { useState } from "react";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import { useRouter } from "next/router";
+
+export function DeletePromptVersion({
+  promptVersionId,
+  projectId,
+  version,
+  countVersions,
+}: {
+  promptVersionId: string;
+  projectId: string;
+  version: number;
+  countVersions: number;
+}) {
+  const utils = api.useUtils();
+  const router = useRouter();
+  const [isOpen, setIsOpen] = useState(false);
+  const hasAccess = useHasAccess({ projectId, scope: "prompts:CUD" });
+
+  const mutDeletePromptVersion = api.prompts.deleteVersion.useMutation({
+    onSuccess: () => {
+      void utils.prompts.invalidate();
+      if (countVersions > 1) {
+        void router.replace(
+          {
+            pathname: router.pathname,
+            query: { ...router.query, version: undefined },
+          },
+          undefined,
+          { shallow: true },
+        );
+      } else {
+        void router.push(`/project/${projectId}/prompts`);
+      }
+    },
+  });
+
+  if (!hasAccess) {
+    return null;
+  }
+
+  return (
+    <Popover
+      key={promptVersionId}
+      open={isOpen}
+      onOpenChange={() => setIsOpen(!isOpen)}
+    >
+      <PopoverTrigger asChild>
+        <Button variant="outline" type="button" size="icon">
+          <Trash2 className="h-5 w-5" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent>
+        <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
+        <p className="mb-3 text-sm">
+          This action deletes the prompt version. Requests of version{" "}
+          <code className="relative rounded bg-muted px-[0.3rem] py-[0.2rem] font-mono text-sm font-semibold">
+            {version}
+          </code>
+          of this prompt will return an error.
+        </p>
+        <div className="flex justify-end space-x-4">
+          <Button
+            type="button"
+            variant="destructive"
+            loading={mutDeletePromptVersion.isLoading}
+            onClick={() => {
+              void mutDeletePromptVersion.mutateAsync({
+                promptVersionId,
+                projectId,
+              });
+              setIsOpen(false);
+            }}
+          >
+            Delete Prompt Version
+          </Button>
+        </div>
+      </PopoverContent>
+    </Popover>
+  );
+}

src/features/prompts/components/delete-prompt.tsx

@@ -0,0 +1,69 @@
+import { Button } from "@/src/components/ui/button";
+import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { api } from "@/src/utils/api";
+import { Trash2 } from "lucide-react";
+import { useState } from "react";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+
+export function DeletePrompt({
+  projectId,
+  promptName,
+}: {
+  projectId: string;
+  promptName: string;
+}) {
+  const utils = api.useUtils();
+  const [isOpen, setIsOpen] = useState(false);
+  const hasAccess = useHasAccess({ projectId, scope: "prompts:CUD" });
+
+  const mutDeletePrompt = api.prompts.delete.useMutation({
+    onSuccess: () => {
+      void utils.prompts.invalidate();
+    },
+  });
+
+  if (!hasAccess) {
+    return null;
+  }
+
+  return (
+    <Popover open={isOpen} onOpenChange={() => setIsOpen(!isOpen)}>
+      <PopoverTrigger asChild>
+        <Button variant="ghost" size="xs">
+          <Trash2 className="h-4 w-4" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent>
+        <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
+        <p className="mb-3 text-sm">
+          This action permanently deletes this prompt. All requests to fetch
+          prompt{" "}
+          <code className="relative rounded bg-muted px-[0.3rem] py-[0.2rem] font-mono text-sm font-semibold">
+            {promptName}
+          </code>{" "}
+          will error.
+        </p>
+        <div className="flex justify-end space-x-4">
+          <Button
+            type="button"
+            variant="destructive"
+            loading={mutDeletePrompt.isLoading}
+            onClick={() => {
+              void mutDeletePrompt.mutateAsync({
+                projectId,
+                promptName,
+              });
+              setIsOpen(false);
+            }}
+          >
+            Delete Prompt
+          </Button>
+        </div>
+      </PopoverContent>
+    </Popover>
+  );
+}

src/features/prompts/components/new-prompt-button.tsx

@@ -77,13 +77,13 @@ const formSchema = z.object({
       const variables = extractVariables(val);
       const matches = variables.map((variable) => {
         // check regex here
-        if (variable.match(/^[A-Za-z]+$/)) {
+        if (variable.match(/^[A-Za-z_]+$/)) {
           return true;
         }
         return false;
       });
       return !matches.includes(false);
-    }, "Variables must only contain letters"),
+    }, "Variables must only contain letters and underscores (_)"),
   isActive: z.boolean({
     required_error: "Enter whether the prompt should go live",
   }),
@@ -153,11 +153,9 @@ export const NewPromptForm = (props: {
         props.onFormSuccess?.();
         form.reset();
         // go to the following page after creating the prompt
-        if (newPrompt) {
-          void router.push(
-            `/project/${props.projectId}/prompts/${newPrompt.name}`,
-          );
-        }
+        void router.push(
+          `/project/${props.projectId}/prompts/${newPrompt.name}`,
+        );
       })
       .catch((error) => {
         console.error(error);

src/features/prompts/components/prompt-detail.tsx

@@ -14,6 +14,7 @@ import { PromotePrompt } from "@/src/features/prompts/components/promote-prompt"
 import { ScrollArea } from "@radix-ui/react-scroll-area";
 import { useQueryParam, NumberParam } from "use-query-params";
 import router from "next/router";
+import { DeletePromptVersion } from "@/src/features/prompts/components/delete-prompt-version";
 
 export type PromptDetailProps = {
   projectId: string;
@@ -77,6 +78,12 @@ export const PromptDetail = (props: PromptDetailProps) => {
                     <Pencil className="h-5 w-5" />
                   </Button>
                 </CreatePromptDialog>
+                <DeletePromptVersion
+                  projectId={props.projectId}
+                  promptVersionId={prompt.id}
+                  version={prompt.version}
+                  countVersions={promptHistory.data.length}
+                />
                 <DetailPageNav
                   key="nav"
                   currentId={prompt.name}

src/features/prompts/components/prompts-table.tsx

@@ -5,6 +5,7 @@ import { Button } from "@/src/components/ui/button";
 import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
 import { CreatePromptDialog } from "@/src/features/prompts/components/new-prompt-button";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
+import { DeletePrompt } from "@/src/features/prompts/components/delete-prompt";
 
 import { api } from "@/src/utils/api";
 import { type RouterOutput } from "@/src/utils/types";
@@ -71,6 +72,18 @@ export function PromptTable(props: { projectId: string }) {
         return createdAt.toLocaleString();
       },
     },
+    {
+      accessorKey: "actions",
+      header: "Actions",
+      cell: ({ row }) => {
+        return (
+          <DeletePrompt
+            projectId={props.projectId}
+            promptName={row.getValue("name")}
+          />
+        );
+      },
+    },
   ];
 
   const convertToTableRow = (

src/features/prompts/server/prompt-router.ts

@@ -6,6 +6,7 @@ import {
 } from "@/src/server/api/trpc";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import { type Prompt, type PrismaClient } from "@prisma/client";
+import { auditLog } from "@/src/features/audit-logs/auditLog";
 
 export const CreatePrompt = z.object({
   projectId: z.string(),
@@ -70,7 +71,7 @@ export const promptRouter = createTRPCRouter({
           scope: "prompts:CUD",
         });
 
-        return await createPrompt({
+        const prompt = await createPrompt({
           projectId: input.projectId,
           name: input.name,
           prompt: input.prompt,
@@ -78,6 +79,116 @@ export const promptRouter = createTRPCRouter({
           createdBy: ctx.session.user.id,
           prisma: ctx.prisma,
         });
+
+        if (!prompt) {
+          throw new Error("Failed to create prompt");
+        }
+
+        await auditLog(
+          {
+            session: ctx.session,
+            resourceType: "prompt",
+            resourceId: prompt.id,
+            action: "create",
+            after: prompt,
+          },
+          ctx.prisma,
+        );
+
+        return prompt;
+      } catch (e) {
+        console.log(e);
+        throw e;
+      }
+    }),
+  delete: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        promptName: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "prompts:CUD",
+        });
+
+        // fetch prompts before deletion to enable audit logging
+        const prompts = await ctx.prisma.prompt.findMany({
+          where: {
+            projectId: input.projectId,
+            name: input.promptName,
+          },
+        });
+
+        for (const prompt of prompts) {
+          await auditLog(
+            {
+              session: ctx.session,
+              resourceType: "prompt",
+              resourceId: prompt.id,
+              action: "delete",
+              before: prompt,
+            },
+            ctx.prisma,
+          );
+        }
+
+        await ctx.prisma.prompt.deleteMany({
+          where: {
+            projectId: input.projectId,
+            id: {
+              in: prompts.map((p) => p.id),
+            },
+          },
+        });
+      } catch (e) {
+        console.log(e);
+        throw e;
+      }
+    }),
+  deleteVersion: protectedProjectProcedure
+    .input(
+      z.object({
+        promptVersionId: z.string(),
+        projectId: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      try {
+        throwIfNoAccess({
+          session: ctx.session,
+          projectId: input.projectId,
+          scope: "prompts:CUD",
+        });
+
+        const promptVersion = await ctx.prisma.prompt.findFirstOrThrow({
+          where: {
+            id: input.promptVersionId,
+            projectId: input.projectId,
+          },
+        });
+
+        await auditLog(
+          {
+            session: ctx.session,
+            resourceType: "prompt",
+            resourceId: input.promptVersionId,
+            action: "delete",
+            before: promptVersion,
+          },
+          ctx.prisma,
+        );
+
+        await ctx.prisma.prompt.delete({
+          where: {
+            id: input.promptVersionId,
+            projectId: input.projectId,
+          },
+        });
       } catch (e) {
         console.log(e);
         throw e;
@@ -99,6 +210,20 @@ export const promptRouter = createTRPCRouter({
           },
         });
 
+        await auditLog(
+          {
+            session: ctx.session,
+            resourceType: "prompt",
+            resourceId: toBePromotedPrompt.id,
+            action: "promote",
+            after: {
+              ...toBePromotedPrompt,
+              isActive: true,
+            },
+          },
+          ctx.prisma,
+        );
+
         const latestActivePrompt = await ctx.prisma.prompt.findFirst({
           where: {
             projectId: input.projectId,

src/features/public-api/components/CreateApiKeyButton.tsx

@@ -11,7 +11,7 @@ import {
 import { CodeView } from "@/src/components/ui/code";
 import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
 import { usePostHog } from "posthog-js/react";
-import { env } from "@/src/env.mjs";
+import { QuickstartExamples } from "@/src/features/public-api/components/QuickstartExamples";
 
 export function CreateApiKeyButton(props: { projectId: string }) {
   const utils = api.useUtils();
@@ -21,8 +21,7 @@ export function CreateApiKeyButton(props: { projectId: string }) {
     scope: "apiKeys:create",
   });
 
-  const hostname =
-    env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== "EU" ? window.origin : undefined;
+  const hostname = window.origin;
 
   const mutCreateApiKey = api.apiKeys.create.useMutation({
     onSuccess: () => utils.apiKeys.invalidate(),
@@ -66,28 +65,15 @@ export function CreateApiKeyButton(props: { projectId: string }) {
           Create new API keys
         </Button>
       </DialogTrigger>
-      <DialogContent onPointerDownOutside={(e) => e.preventDefault()}>
-        {hostname ? (
-          <>
-            <DialogTitle>Hostname</DialogTitle>
-            <div className="mb-6">
-              <div className="my-2">
-                When connecting to Langfuse, use this hostname / baseurl.
-              </div>
-              <CodeView content={hostname} />
-            </div>
-          </>
-        ) : null}
-
+      <DialogContent
+        onPointerDownOutside={(e) => e.preventDefault()}
+        className="max-w-full md:max-w-xl"
+      >
         <DialogTitle>API Keys</DialogTitle>
         <div className="mb-2">
           <div className="text-md font-semibold">Secret Key</div>
           <div className="my-2">
-            Please save this secret key.{" "}
-            <span className="font-semibold">
-              You will not be able to view it again
-            </span>
-            . If you lose it, you will need to generate a new one.
+            This key can only be viewed once. You can always generate a new key.
           </div>
           <CodeView content={generatedKeys?.secretKey ?? "Loading ..."} />
         </div>
@@ -95,6 +81,20 @@ export function CreateApiKeyButton(props: { projectId: string }) {
           <div className="text-md mb-2 font-semibold">Public Key</div>
           <CodeView content={generatedKeys?.publicKey ?? "Loading ..."} />
         </div>
+        <div>
+          <div className="text-md mb-2 font-semibold">Host</div>
+          <CodeView content={hostname} />
+        </div>
+        {generatedKeys && (
+          <div className="mb-2">
+            <div className="text-md my-2 font-semibold">Usage</div>
+            <QuickstartExamples
+              secretKey={generatedKeys.secretKey}
+              publicKey={generatedKeys.publicKey}
+              host={hostname}
+            />
+          </div>
+        )}
       </DialogContent>
     </Dialog>
   );

src/features/public-api/components/QuickstartExamples.tsx

@@ -0,0 +1,210 @@
+import { CodeView } from "@/src/components/ui/code";
+import {
+  Tabs,
+  TabsList,
+  TabsContent,
+  TabsTrigger,
+} from "@/src/components/ui/tabs";
+
+export const QuickstartExamples = ({
+  secretKey,
+  publicKey,
+  host,
+}: {
+  secretKey: string;
+  publicKey: string;
+  host: string;
+}) => (
+  <Tabs defaultValue="python">
+    <TabsList>
+      <TabsTrigger value="python">Python</TabsTrigger>
+      <TabsTrigger value="js">JS/TS</TabsTrigger>
+      <TabsTrigger value="openai">OpenAI</TabsTrigger>
+      <TabsTrigger value="langchain">Langchain</TabsTrigger>
+      <TabsTrigger value="langchain-js">Langchain JS</TabsTrigger>
+      <TabsTrigger value="other">Other</TabsTrigger>
+    </TabsList>
+    <TabsContent value="python">
+      <CodeView content="pip install langfuse" className="mb-2 bg-blue-50" />
+      <CodeView
+        className="bg-blue-50"
+        content={`from langfuse import Langfuse\n\nlangfuse = Langfuse(\n  secret_key="${secretKey}",\n  public_key="${publicKey}",\n  host="${host}"\n)`}
+      />
+      <p className="mt-3 text-xs text-gray-600">
+        See{" "}
+        <a
+          href="https://langfuse.com/docs/get-started"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          Quickstart
+        </a>{" "}
+        and{" "}
+        <a
+          href="https://langfuse.com/docs/sdk/python"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          Python docs
+        </a>{" "}
+        for more details.
+      </p>
+    </TabsContent>
+    <TabsContent value="js">
+      <CodeView content="npm install langfuse" className="mb-2 bg-blue-50" />
+      <CodeView
+        className="bg-blue-50"
+        content={`import { Langfuse } from "langfuse";\n\nconst langfuse = new Langfuse({\n  secretKey: "${secretKey}",\n  publicKey: "${publicKey}",\n  baseUrl: "${host}"\n});`}
+      />
+      <p className="mt-3 text-xs text-gray-600">
+        See{" "}
+        <a
+          href="https://langfuse.com/docs/get-started"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          Quickstart
+        </a>{" "}
+        and{" "}
+        <a
+          href="https://langfuse.com/docs/sdk/typescript"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          JS/TS docs
+        </a>{" "}
+        for more details.
+      </p>
+    </TabsContent>
+    <TabsContent value="openai">
+      <p className="mt-2 text-xs text-gray-600">
+        The integration is a drop-in replacement for the OpenAI Python SDK. By
+        changing the import, Langfuse will capture all LLM calls and send them
+        to Langfuse asynchronously.
+      </p>
+      <CodeView content="pip install langfuse" className="my-2 bg-blue-50" />
+      <CodeView
+        title=".env"
+        content={`LANGFUSE_SECRET_KEY=${secretKey};\nLANGFUSE_PUBLIC_KEY=${publicKey};\nLANGFUSE_HOST="${host}";`}
+        className="my-2 bg-blue-50"
+      />
+      <CodeView
+        content={`# remove: import openai\n\nfrom langfuse.openai import openai`}
+        className="my-2 bg-blue-50"
+      />
+      <p className="mt-2 text-xs text-gray-600">
+        Use the OpenAI SDK as you would normally. See the{" "}
+        <a
+          href="https://langfuse.com/docs/integrations/openai"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          OpenAI Integration docs
+        </a>{" "}
+        for more details.
+      </p>
+    </TabsContent>
+    <TabsContent value="langchain">
+      <p className="mt-2 text-xs text-gray-600">
+        The integration uses the Langchain callback system to automatically
+        capture detailed traces of your Langchain executions.
+      </p>
+      <CodeView content="pip install langfuse" className="my-2 bg-blue-50" />
+      <CodeView
+        title=".env"
+        content={`LANGFUSE_SECRET_KEY=${secretKey};\nLANGFUSE_PUBLIC_KEY=${publicKey};\nLANGFUSE_HOST="${host}";`}
+        className="my-2 bg-blue-50"
+      />
+      <CodeView content={LANGCHAIN_PYTHON_CODE} className="my-2 bg-blue-50" />
+      <p className="mt-2 text-xs text-gray-600">
+        See the{" "}
+        <a
+          href="https://langfuse.com/docs/integrations/langchain/python"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          Langchain Integration docs
+        </a>{" "}
+        for more details.
+      </p>
+    </TabsContent>
+    <TabsContent value="langchain-js">
+      <p className="mt-2 text-xs text-gray-600">
+        The integration uses the Langchain callback system to automatically
+        capture detailed traces of your Langchain executions.
+      </p>
+      <CodeView
+        content="npm install langfuse-langchain"
+        className="my-2 bg-blue-50"
+      />
+      <CodeView
+        title=".env"
+        content={`LANGFUSE_SECRET_KEY=${secretKey};\nLANGFUSE_PUBLIC_KEY=${publicKey};\LANGFUSE_BASEURL="${host}";`}
+        className="my-2 bg-blue-50"
+      />
+      <CodeView content={LANGCHAIN_JS_CODE} className="my-2 bg-blue-50" />
+      <p className="mt-2 text-xs text-gray-600">
+        See the{" "}
+        <a
+          href="https://langfuse.com/docs/integrations/langchain/typescript"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          Langchain Integration docs
+        </a>{" "}
+        for more details.
+      </p>
+    </TabsContent>
+    <TabsContent value="other">
+      <p className="mt-2 text-xs text-gray-600">
+        Use the{" "}
+        <a
+          href="https://api.reference.langfuse.com/"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          API
+        </a>{" "}
+        or one of the{" "}
+        <a
+          href="https://langfuse.com/docs/integrations"
+          className="underline"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          native integrations
+        </a>{" "}
+        (e.g. LiteLLM, Flowise, and Langflow) to integrate with Langfuse.
+      </p>
+    </TabsContent>
+  </Tabs>
+);
+
+const LANGCHAIN_PYTHON_CODE = `from langfuse.callback import CallbackHandler
+
+# <Your Langchain code here>
+ 
+# Add handler to run/invoke/call/chat
+agent.run("<user_input>", callbacks=[handler])`;
+
+const LANGCHAIN_JS_CODE = `import { CallbackHandler } from "langfuse-langchain";
+ 
+// Initialize Langfuse callback handler
+const handler = new CallbackHandler();
+ 
+// Your Langchain implementation
+const chain = new LLMChain(...);
+ 
+// Add handler as callback when running the Langchain agent
+await chain.call(
+  { input: "<user_input>" },
+  { callbacks: [handler] }
+);`;

src/features/public-api/server/apiAuth.ts

@@ -91,7 +91,9 @@ export async function verifyAuthHeaderAndReturnScope(
     console.error("Error verifying auth header: ", error);
     return {
       validKey: false,
-      error: error instanceof Error ? error.message : "Authorization error",
+      error:
+        (error instanceof Error ? error.message : "Authorization error") +
+        ". Confirm that you've configured the correct host.",
     };
   }
   return {

src/features/public-api/server/apiKeyRouter.ts

@@ -1,3 +1,4 @@
+import { auditLog } from "@/src/features/audit-logs/auditLog";
 import { generateKeySet } from "@/src/features/public-api/lib/apiKeys";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import {
@@ -64,6 +65,13 @@ export const apiKeysRouter = createTRPCRouter({
         },
       });
 
+      await auditLog({
+        session: ctx.session,
+        resourceType: "apiKey",
+        resourceId: apiKey.id,
+        action: "create",
+      });
+
       return {
         id: apiKey.id,
         createdAt: apiKey.createdAt,
@@ -86,6 +94,12 @@ export const apiKeysRouter = createTRPCRouter({
         projectId: input.projectId,
         scope: "apiKeys:delete",
       });
+      await auditLog({
+        session: ctx.session,
+        resourceType: "apiKey",
+        resourceId: input.id,
+        action: "delete",
+      });
 
       // Make sure the API key exists and belongs to the project the user has access to
       const apiKey = await ctx.prisma.apiKey.findFirstOrThrow({

src/features/public-api/server/ingestion-api-schema.ts

@@ -1,13 +1,15 @@
-import { ObservationLevel } from "@prisma/client";
-import { jsonSchema } from "@/src/utils/zod";
-import { z } from "zod";
 import lodash from "lodash";
+import { z } from "zod";
+
+import { ModelUsageUnit } from "@/src/constants";
+import { jsonSchema } from "@/src/utils/zod";
+import { ObservationLevel } from "@prisma/client";
 
 export const Usage = z.object({
   input: z.number().int().nullish(),
   output: z.number().int().nullish(),
   total: z.number().int().nullish(),
-  unit: z.enum(["TOKENS", "CHARACTERS"]).nullish(),
+  unit: z.nativeEnum(ModelUsageUnit).nullish(),
   inputCost: z.number().nullish(),
   outputCost: z.number().nullish(),
   totalCost: z.number().nullish(),
@@ -17,7 +19,7 @@ const MixedUsage = z.object({
   input: z.number().int().nullish(),
   output: z.number().int().nullish(),
   total: z.number().int().nullish(),
-  unit: z.enum(["TOKENS", "CHARACTERS"]).nullish(),
+  unit: z.nativeEnum(ModelUsageUnit).nullish(),
   promptTokens: z.number().int().nullish(),
   completionTokens: z.number().int().nullish(),
   totalTokens: z.number().int().nullish(),
@@ -40,25 +42,23 @@ export const usage = MixedUsage.nullish()
         input: v.promptTokens,
         output: v.completionTokens,
         total: v.totalTokens,
-        unit: "TOKENS",
+        unit: ModelUsageUnit.Tokens,
       };
     }
-    // if we get the new generic format, we do not set a default
-    if ("input" in v || "output" in v || "total" in v || "unit" in v) {
-      const unit = v.unit;
-      return { ...v, unit };
-    }
 
     // if the object is empty, we return undefined
     if (lodash.isEmpty(v)) {
       return undefined;
     }
+
+    return v;
   })
   // ensure output is always of new usage model
   .pipe(Usage.nullish());
 
 export const TraceBody = z.object({
   id: z.string().nullish(),
+  timestamp: stringDate,
   name: z.string().nullish(),
   externalId: z.string().nullish(),
   input: jsonSchema.nullish(),

src/features/rbac/server/projectMembersRouter.ts

@@ -1,3 +1,4 @@
+import { auditLog } from "@/src/features/audit-logs/auditLog";
 import { sendProjectInvitation } from "@/src/features/email/lib/project-invitation";
 import { throwIfNoAccess } from "@/src/features/rbac/utils/checkAccess";
 import {
@@ -5,6 +6,7 @@ import {
   protectedProjectProcedure,
 } from "@/src/server/api/trpc";
 import { MembershipRole } from "@prisma/client";
+import { TRPCError } from "@trpc/server";
 import * as z from "zod";
 
 export const projectMembersRouter = createTRPCRouter({
@@ -75,8 +77,7 @@ export const projectMembersRouter = createTRPCRouter({
       if (input.userId === ctx.session.user.id)
         throw new Error("You cannot remove yourself from a project");
 
-      // use deleteMany to protect against deleting owner with where clause
-      return ctx.prisma.membership.deleteMany({
+      const membership = await ctx.prisma.membership.findFirst({
         where: {
           projectId: input.projectId,
           userId: input.userId,
@@ -85,6 +86,26 @@ export const projectMembersRouter = createTRPCRouter({
           },
         },
       });
+
+      if (!membership) throw new TRPCError({ code: "NOT_FOUND" });
+
+      await auditLog({
+        session: ctx.session,
+        resourceType: "membership",
+        resourceId: membership.projectId + "--" + membership.userId,
+        action: "delete",
+        before: membership,
+      });
+
+      // use ids from membership to make sure owners cannot delete themselves
+      return await ctx.prisma.membership.delete({
+        where: {
+          projectId_userId: {
+            projectId: membership.projectId,
+            userId: membership.userId,
+          },
+        },
+      });
     }),
   deleteInvitation: protectedProjectProcedure
     .input(
@@ -100,6 +121,13 @@ export const projectMembersRouter = createTRPCRouter({
         scope: "members:delete",
       });
 
+      await auditLog({
+        session: ctx.session,
+        resourceType: "membershipInvitation",
+        resourceId: input.id,
+        action: "delete",
+      });
+
       return await ctx.prisma.membershipInvitation.delete({
         where: {
           id: input.id,
@@ -132,13 +160,21 @@ export const projectMembersRouter = createTRPCRouter({
         },
       });
       if (user) {
-        return await ctx.prisma.membership.create({
+        const membership = await ctx.prisma.membership.create({
           data: {
             userId: user.id,
             projectId: input.projectId,
             role: input.role,
           },
         });
+        await auditLog({
+          session: ctx.session,
+          resourceType: "membership",
+          resourceId: input.projectId + "--" + user.id,
+          action: "create",
+          after: membership,
+        });
+        return membership;
       } else {
         const invitation = await ctx.prisma.membershipInvitation.create({
           data: {
@@ -148,6 +184,13 @@ export const projectMembersRouter = createTRPCRouter({
             senderId: ctx.session.user.id,
           },
         });
+        await auditLog({
+          session: ctx.session,
+          resourceType: "membershipInvitation",
+          resourceId: invitation.id,
+          action: "create",
+          after: invitation,
+        });
 
         const project = await ctx.prisma.project.findFirst({
           where: {

src/features/rbac/utils/checkAccess.ts

@@ -5,20 +5,24 @@ import {
 import { type MembershipRole } from "@prisma/client";
 import { TRPCError } from "@trpc/server";
 import { type Session } from "next-auth";
-import { useSession, type SessionContextValue } from "next-auth/react";
+import { useSession } from "next-auth/react";
 
 type HasAccessParams =
   | {
       role: MembershipRole;
       scope: Scope;
+      admin?: boolean; // prop user.admin
     }
   | {
-      session: SessionContextValue | Session;
+      session: null | Session;
       projectId: string;
       scope: Scope;
     };
 
-// For use in TRPC routes
+/**
+ * Check if user has access to the given scope, for use in TRPC resolvers
+ * @throws TRPCError("UNAUTHORIZED") if user does not have access
+ */
 export const throwIfNoAccess = (p: HasAccessParams) => {
   if (!hasAccess(p))
     throw new TRPCError({
@@ -28,27 +32,26 @@ export const throwIfNoAccess = (p: HasAccessParams) => {
     });
 };
 
-// For use in UI components as react hook
+/**
+ * React hook to check if user has access to the given scope
+ * @returns true if user has access, false otherwise or while loading
+ */
 export const useHasAccess = (p: { projectId: string; scope: Scope }) => {
   const session = useSession();
-  return hasAccess({ session, ...p });
+  return hasAccess({ session: session.data, ...p });
 };
 
 // For use in UI components as function, if session is already available
 export function hasAccess(p: HasAccessParams): boolean {
-  const role: MembershipRole | undefined =
-    "role" in p
-      ? // MembershipRole
-        p.role
-      : "data" in p.session
-        ? // SessionContextValue
-          p.session.data?.user?.projects.find(
-            (project) => project.id === p.projectId,
-          )?.role
-        : // Session
-          p.session.user?.projects.find((project) => project.id === p.projectId)
-            ?.role;
-  if (role === undefined) return false;
+  const isAdmin = "role" in p ? p.admin : p.session?.user?.admin;
+  if (isAdmin && p.scope.endsWith(":read")) return true;
 
-  return roleAccessRights[role].includes(p.scope);
+  const projectRole: MembershipRole | undefined =
+    "role" in p
+      ? p.role
+      : p.session?.user?.projects.find((project) => project.id === p.projectId)
+          ?.role;
+  if (projectRole === undefined) return false;
+
+  return roleAccessRights[projectRole].includes(p.scope);
 }