Compare commits

...
39 Commits
Author SHA1 Message Date
Max Deichmann e4a029988f chore: release v2.9.0
CI/CD / lint (push) Waiting to run
CI/CD / test-docker-build (push) Waiting to run
CI/CD / tests (18) (push) Waiting to run
CI/CD / tests (20) (push) Waiting to run
CI/CD / e2e-tests (push) Waiting to run
CI/CD / all-ci-passed (push) Blocked by required conditions
CI/CD / push-docker-image (push) Blocked by required conditions
release.yml / release (push) Waiting to run
2024-02-29 14:50:13 +01:00
Marc Klingen d8bb7703ae fix(ui): second level headings should be h3 2024-02-29 12:24:55 +01:00
Marc Klingen bd7574411e chore: add npm run dx to root 2024-02-29 12:02:24 +01:00
Marc Klingen 2f55acec7c docs: update readme 2024-02-28 23:15:42 +01:00
SebandGitHub 78207ba274 chore: improve date picker formatting (#1283) 2024-02-28 20:14:16 +01:00
SebandGitHub 2250be58a5 docs: fix link to db schema (#1282) 2024-02-28 20:12:49 +01:00
Max DeichmannandGitHub 338517828c perf: increase sample rate for POST and trpc (#1284) 2024-02-28 19:06:10 +00:00
Max DeichmannandGitHub 2e80d4b475 perf: improve dashboard performance (#1281) 2024-02-28 16:51:38 +01:00
Marc KlingenandGitHub 45c9af76d8 fix(api): add input, output, sessionId, public to GET /traces (#1280) 2024-02-28 13:47:40 +00:00
Max DeichmannandGitHub bc27618c0e perf: improve generations table performance (#1279) 2024-02-28 13:41:33 +00:00
Max DeichmannandGitHub a4c8f0b702 perf: improve traces table performance (#1276) 2024-02-28 13:04:11 +00:00
Max DeichmannandGitHub 787eb90c54 perf: improve observations view and all traces (#1275) 2024-02-28 09:48:20 +00:00
Max DeichmannandGitHub 9b4601debf refactor: add load database setup script (#1273) 2024-02-27 23:43:30 +00:00
Max DeichmannandGitHub fcc0ee922f refactor: add sentry users to API (#1269) 2024-02-27 16:17:37 +01:00
Max DeichmannandGitHub 3271c4abd0 fix: observations index (#1267) 2024-02-27 14:33:56 +01:00
Max DeichmannandGitHub 2dbb9fec87 fix: fix observations index (#1266) 2024-02-27 12:42:32 +01:00
e7438c05b8 feat(generation): extend generations table (#1216)
Co-authored-by: Max Deichmann <m.deichmann@tum.de>
2024-02-27 11:20:27 +00:00
Marc Klingen 45fcf8902b fix(self-host): event mapping 2024-02-27 02:40:54 +01:00
Max DeichmannandGitHub a4bfdb25a5 perf: revert traces router query (#1261)
revert
2024-02-26 22:37:35 +00:00
Max DeichmannandGitHub 462e8a7b8c perf: traces table performance improvements (#1256) 2024-02-26 22:57:53 +01:00
Marc KlingenandGitHub 86edd502b3 feat(ui): add alert that only ChatML generations is exported as JSONL (#1257) 2024-02-26 19:56:41 +00:00
Max DeichmannandGitHub 01a7e679fa fix: correctly return 500 and 400 API errors (#1253) 2024-02-26 12:44:03 +00:00
Marc KlingenandGitHub c6406e1ddb fix(ui): sort/filter sessions by total cost (#1254) 2024-02-26 11:35:20 +00:00
Max DeichmannandGitHub 82d4881dcf fix: improve contributing.md (#1252) 2024-02-26 09:23:51 +00:00
Max DeichmannandGitHub 42d3167591 fix: docs and root level commands (#1251) 2024-02-26 08:52:09 +00:00
Marc KlingenandGitHub 977ac39ec5 docs(api): improve api reference descriptions (#1244) 2024-02-23 19:44:11 +00:00
Marc Klingen e4d8db6a34 chore: release v2.8.0
CI/CD / lint (push) Waiting to run
CI/CD / test-docker-build (push) Waiting to run
CI/CD / tests (18) (push) Waiting to run
CI/CD / tests (20) (push) Waiting to run
CI/CD / e2e-tests (push) Waiting to run
CI/CD / all-ci-passed (push) Blocked by required conditions
CI/CD / push-docker-image (push) Blocked by required conditions
release.yml / release (push) Waiting to run
2024-02-23 20:23:00 +01:00
Marc KlingenandGitHub a39bbf541d feat(api): add GET /score/[scoreId] (#1242) 2024-02-23 19:20:27 +00:00
Max DeichmannandGitHub eef5218dbd refactor: add worker tests (#1241) 2024-02-23 17:44:54 +00:00
Marc KlingenandGitHub b5e802171b fix(api): deduplicate score and observation ids in GET /traces (#1240) 2024-02-23 17:38:55 +00:00
Max DeichmannandGitHub 96c4f28b5b refactor: linting worker (#1239) 2024-02-23 17:34:16 +00:00
Max DeichmannandGitHub 8ad4a46850 feat: add database connection to worker (#1238) 2024-02-23 16:55:16 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>Max Deichmann
d8f2d6cf4e chore(deps): bump the prisma group in /web with 2 updates (#1231)
Bumps the prisma group in /web with 2 updates: [@prisma/client](https://github.com/prisma/prisma/tree/HEAD/packages/client) and [prisma](https://github.com/prisma/prisma/tree/HEAD/packages/cli).


Updates `@prisma/client` from 5.9.1 to 5.10.2
- [Release notes](https://github.com/prisma/prisma/releases)
- [Commits](https://github.com/prisma/prisma/commits/5.10.2/packages/client)

Updates `prisma` from 5.9.1 to 5.10.2
- [Release notes](https://github.com/prisma/prisma/releases)
- [Commits](https://github.com/prisma/prisma/commits/5.10.2/packages/cli)

---
updated-dependencies:
- dependency-name: "@prisma/client"
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: prisma
- dependency-name: prisma
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: prisma
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Max Deichmann <m.deichmann@tum.de>
2024-02-23 16:50:10 +00:00
b8639ebb3b feat: use Json editor for dataset item editor (#1171)
Co-authored-by: gitstart-langfuse <gitstart-langfuse@users.noreply.github.com>
Co-authored-by: Max Deichmann <m.deichmann@tum.de>
2024-02-23 14:46:27 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
8be73970c2 chore(deps): bump the sentry group in /web with 3 updates (#1230)
Bumps the sentry group in /web with 3 updates: [@sentry/nextjs](https://github.com/getsentry/sentry-javascript), [@sentry/profiling-node](https://github.com/getsentry/sentry-javascript) and [@sentry/types](https://github.com/getsentry/sentry-javascript).


Updates `@sentry/nextjs` from 7.101.1 to 7.102.1
- [Release notes](https://github.com/getsentry/sentry-javascript/releases)
- [Changelog](https://github.com/getsentry/sentry-javascript/blob/7.102.1/CHANGELOG.md)
- [Commits](https://github.com/getsentry/sentry-javascript/compare/7.101.1...7.102.1)

Updates `@sentry/profiling-node` from 7.101.1 to 7.102.1
- [Release notes](https://github.com/getsentry/sentry-javascript/releases)
- [Changelog](https://github.com/getsentry/sentry-javascript/blob/7.102.1/CHANGELOG.md)
- [Commits](https://github.com/getsentry/sentry-javascript/compare/7.101.1...7.102.1)

Updates `@sentry/types` from 7.101.1 to 7.102.1
- [Release notes](https://github.com/getsentry/sentry-javascript/releases)
- [Changelog](https://github.com/getsentry/sentry-javascript/blob/7.102.1/CHANGELOG.md)
- [Commits](https://github.com/getsentry/sentry-javascript/compare/7.101.1...7.102.1)

---
updated-dependencies:
- dependency-name: "@sentry/nextjs"
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: sentry
- dependency-name: "@sentry/profiling-node"
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: sentry
- dependency-name: "@sentry/types"
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: sentry
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-02-23 14:34:10 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>Max Deichmann
e18ab26bb5 chore(deps): bump @aws-sdk/lib-storage from 3.511.0 to 3.515.0 in /web (#1233)
Bumps [@aws-sdk/lib-storage](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/lib/lib-storage) from 3.511.0 to 3.515.0.
- [Release notes](https://github.com/aws/aws-sdk-js-v3/releases)
- [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/lib/lib-storage/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.515.0/lib/lib-storage)

---
updated-dependencies:
- dependency-name: "@aws-sdk/lib-storage"
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Max Deichmann <m.deichmann@tum.de>
2024-02-23 14:28:52 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>Max Deichmann
095478869d chore(deps-dev): bump release-it from 17.0.5 to 17.1.1 in /web (#1234)
Bumps [release-it](https://github.com/release-it/release-it) from 17.0.5 to 17.1.1.
- [Release notes](https://github.com/release-it/release-it/releases)
- [Changelog](https://github.com/release-it/release-it/blob/main/CHANGELOG.md)
- [Commits](https://github.com/release-it/release-it/compare/17.0.5...17.1.1)

---
updated-dependencies:
- dependency-name: release-it
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Max Deichmann <m.deichmann@tum.de>
2024-02-23 14:23:04 +00:00
Marc KlingenandGitHub 4db7dab8ae fix(export): messages key in jsonl output format (#1236) 2024-02-23 13:54:31 +00:00
Max DeichmannandGitHub 086d26b041 fix: fix dependabot in new repo structure (#1228) 2024-02-23 09:35:32 +00:00
95 changed files with 5178 additions and 960 deletions
+28 -1
View File
@@ -6,7 +6,34 @@
version: 2
updates:
- package-ecosystem: npm
directory: "/" # Location of package manifests
directory: "/worker" # Location of package manifests
schedule:
interval: "daily"
rebase-strategy: "disabled" # use dependabot-rebase-stale
commit-message:
prefix: chore
prefix-development: chore
include: scope
ignore:
- dependency-name: "@types/node"
- dependency-name: "@trpc/*"
groups:
sentry:
patterns:
- "@sentry/*"
prisma:
patterns:
- "prisma"
- "@prisma/*"
next:
patterns:
- "eslint-config-next"
- "next"
patches:
update-types:
- "patch"
- package-ecosystem: npm
directory: "/web" # Location of package manifests
schedule:
interval: "daily"
rebase-strategy: "disabled" # use dependabot-rebase-stale
+8 -2
View File
@@ -18,6 +18,9 @@ defaults:
jobs:
lint:
defaults:
run:
working-directory: .
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
@@ -30,10 +33,13 @@ jobs:
run: |
npm ci
- name: Load default env
working-directory: .
run: |
cp .env.dev.example .env
- name: lint
- name: lint web
working-directory: ./web
run: npm run lint
- name: lint worker
working-directory: ./worker
run: npm run lint
test-docker-build:
+11 -2
View File
@@ -91,7 +91,7 @@ flowchart TB
The diagram below may not show all relationships if the foreign key is not defined in the database schema. For instance, `trace_id` in the `observation` table is not defined as a foreign key to the `trace` table to allow unordered ingestion of these objects, but it is still a foreign key in the application code.
Full database schema: [prisma/schema.prisma](prisma/schema.prisma)
Full database schema: [web/prisma/schema.prisma](web/prisma/schema.prisma)
<img src="./web/prisma/database.svg">
@@ -110,7 +110,14 @@ flowchart LR
App --- DB
```
## Development Setup
## Repository Structure
This repository contains the following packages:
- `web`: is the main application package providing Frontend and Backend APIs for Langfuse
- `worker` (no production yet): contains an application for asynchronous processing of tasks. This package is not yet used in production.
## Development Setup (web package only)
Requirements
@@ -146,6 +153,8 @@ Requirements
6. Run the migrations
The `schema.prisma` file is in the `web` directory and hence, migrations are applied from there.
```bash
npm run db:migrate
+13 -11
View File
@@ -119,20 +119,22 @@ See our the [→ Quickstart](https://langfuse.com/docs/get-started) to get start
### Integrations
| Integration | Supports | Description |
| -------------------------------------------------------- | ------------- | ------------------------------------------------------------------------------- |
| [**SDK** - _recommended_](https://langfuse.com/docs/sdk) | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
| [OpenAI](https://langfuse.com/docs/openai) | Python | Automated instrumentation using drop-in replacement of OpenAI SDK. |
| [Langchain](https://langfuse.com/docs/langchain) | Python, JS/TS | Automated instrumentation by passing callback handler to Langchain application. |
| [API](https://langfuse.com/docs/api) | | Directly call the public API. OpenAPI spec available. |
| Integration | Supports | Description |
| ---------------------------------------------------------------- | ------------- | ---------------------------------------------------------------- |
| [**SDK** - _recommended_](https://langfuse.com/docs/sdk) | Python, JS/TS | Manual instrumentation using the SDKs for full flexibility. |
| [OpenAI SDK](https://langfuse.com/docs/integrations/openai) | Python | Automated instrumentation via drop-in replacement of OpenAI SDK. |
| [Langchain](https://langfuse.com/docs/integrations/langchain) | Python, JS/TS | Automated instrumentation via LlamaIndex callback system. |
| [LlamaIndex](https://langfuse.com/docs/integrations/llama-index) | Python, JS/TS | Automated instrumentation via LlamaIndex callback system. |
| [API](https://langfuse.com/docs/api) | | Directly call the public API. OpenAPI spec available. |
External projects/packages that integrate with Langfuse:
| Name | Description |
| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
| [LiteLLM](/https://langfuse.comdocs/litellm) | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
| [Flowise](https://langfuse.com/docs/flowise) | JS/TS no-code builder for customized LLM flows. |
| [Langflow](https://langfuse.com/docs/langflow) | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
| Name | Description |
| --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
| [LiteLLM](https://langfuse.com/docs/integrations/litellm) | Use any LLM as a drop in replacement for GPT. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs). |
| [Flowise](https://langfuse.com/docs/integrations/flowise) | JS/TS no-code builder for customized LLM flows. |
| [Langflow](https://langfuse.com/docs/integrations/langflow) | Python-based UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. |
| [Superagent](https://langfuse.com/docs/integrations/superagent) | Open Source AI Assistant Framework & API for prototyping and deployment of agents. |
## Questions and feedback
+21 -6
View File
@@ -90,14 +90,28 @@ types:
latency: optional<double>
Usage:
docs: Standard interface for usage and cost
properties:
input: optional<integer>
output: optional<integer>
total: optional<integer>
input:
docs: Number of input units (e.g. tokens)
type: optional<integer>
output:
docs: Number of output units (e.g. tokens)
type: optional<integer>
total:
docs: Defaults to input+output if not set
type: optional<integer>
unit: optional<ModelUsageUnit>
inputCost: optional<double>
outputCost: optional<double>
totalCost: optional<double>
inputCost:
docs: USD input cost
type: optional<double>
outputCost:
docs: USD output cost
type: optional<double>
totalCost:
docs: USD total cost, defaults to input+output
type: optional<double>
Score:
properties:
id: string
@@ -145,6 +159,7 @@ types:
# Utilities
ModelUsageUnit:
docs: Unit of usage in Langfuse
enum:
- CHARACTERS
- TOKENS
@@ -7,12 +7,12 @@ service:
endpoints:
create:
method: POST
docs: Create a dataset item, upserts on id
docs: Create a dataset item
path: /dataset-items
request: CreateDatasetItemRequest
response: commons.DatasetItem
get:
docs: Get a specific dataset item
docs: Get a dataset item
method: GET
path: /dataset-items/{id}
path-parameters:
@@ -26,4 +26,6 @@ types:
datasetName: string
input: unknown
expectedOutput: optional<unknown>
id: optional<string>
id:
type: optional<string>
docs: Dataset items are upserted on their id
+8 -3
View File
@@ -7,7 +7,7 @@ service:
base-path: /api/public
endpoints:
batch:
docs: Ingest multiple events to Langfuse
docs: Batched ingestion for Langfuse Tracing
method: POST
path: /ingestion
request:
@@ -31,8 +31,12 @@ types:
sdk-log: SDKLogEvent
# both are legacy
observation-create: CreateObservationEvent
observation-update: UpdateObservationEvent
observation-create:
type: CreateObservationEvent
docs: Deprecated event type
observation-update:
type: UpdateObservationEvent
docs: Deprecated event type
ObservationType:
enum:
@@ -47,6 +51,7 @@ types:
- OpenAIUsage
OpenAIUsage:
docs: Usage interface of OpenAI for improved compatibility.
properties:
promptTokens: optional<integer>
completionTokens: optional<integer>
+1 -1
View File
@@ -7,7 +7,7 @@ service:
base-path: /api/public
endpoints:
get:
docs: Get a specific observation
docs: Get a observation
method: GET
path: /observations/{observationId}
path-parameters:
+1
View File
@@ -7,6 +7,7 @@ service:
base-path: /api/public
endpoints:
get:
docs: Get Project associated with API key
method: GET
path: /projects
response: Projects
+5 -3
View File
@@ -7,7 +7,7 @@ service:
base-path: /api/public
endpoints:
get:
docs: Get a specific prompt
docs: Get a prompt
method: GET
path: /prompts
request:
@@ -17,7 +17,7 @@ service:
version: optional<integer>
response: Prompt
create:
docs: Create a specific prompt
docs: Create a prompt
method: POST
path: /prompts
request: CreatePromptRequest
@@ -27,7 +27,9 @@ types:
CreatePromptRequest:
properties:
name: string
isActive: boolean
isActive:
docs: Should the prompt be promoted to production immediately?
type: boolean
prompt: string
config: optional<unknown>
+11 -2
View File
@@ -7,13 +7,13 @@ service:
base-path: /api/public
endpoints:
create:
docs: Add a score to the database, upserts on id
docs: Create a score
method: POST
path: /scores
request: CreateScoreRequest
response: commons.Score
get:
docs: Get scores
docs: Get a list of scores
method: GET
path: /scores
request:
@@ -24,6 +24,15 @@ service:
userId: optional<string>
name: optional<string>
response: Scores
get-by-id:
docs: Get a score
method: GET
path: /scores/{scoreId}
path-parameters:
scoreId:
type: string
docs: The unique langfuse identifier of a score
response: commons.Score
delete:
docs: Delete a score
method: DELETE
+41 -9
View File
@@ -46,7 +46,7 @@
"_type": "endpoint",
"name": "Create",
"request": {
"description": "Create a dataset item, upserts on id",
"description": "Create a dataset item",
"url": {
"raw": "{{baseUrl}}/api/public/dataset-items",
"host": [
@@ -79,7 +79,7 @@
"_type": "endpoint",
"name": "Get",
"request": {
"description": "Get a specific dataset item",
"description": "Get a dataset item",
"url": {
"raw": "{{baseUrl}}/api/public/dataset-items/:id",
"host": [
@@ -301,7 +301,7 @@
"_type": "endpoint",
"name": "Batch",
"request": {
"description": "Ingest multiple events to Langfuse",
"description": "Batched ingestion for Langfuse Tracing",
"url": {
"raw": "{{baseUrl}}/api/public/ingestion",
"host": [
@@ -400,7 +400,7 @@
"_type": "endpoint",
"name": "Get",
"request": {
"description": "Get a specific observation",
"description": "Get a observation",
"url": {
"raw": "{{baseUrl}}/api/public/observations/:observationId",
"host": [
@@ -500,7 +500,7 @@
"_type": "endpoint",
"name": "Get",
"request": {
"description": null,
"description": "Get Project associated with API key",
"url": {
"raw": "{{baseUrl}}/api/public/projects",
"host": [
@@ -532,7 +532,7 @@
"_type": "endpoint",
"name": "Get",
"request": {
"description": "Get a specific prompt",
"description": "Get a prompt",
"url": {
"raw": "{{baseUrl}}/api/public/prompts?name=&version=",
"host": [
@@ -568,7 +568,7 @@
"_type": "endpoint",
"name": "Create",
"request": {
"description": "Create a specific prompt",
"description": "Create a prompt",
"url": {
"raw": "{{baseUrl}}/api/public/prompts",
"host": [
@@ -608,7 +608,7 @@
"_type": "endpoint",
"name": "Create",
"request": {
"description": "Add a score to the database, upserts on id",
"description": "Create a score",
"url": {
"raw": "{{baseUrl}}/api/public/scores",
"host": [
@@ -641,7 +641,7 @@
"_type": "endpoint",
"name": "Get",
"request": {
"description": "Get scores",
"description": "Get a list of scores",
"url": {
"raw": "{{baseUrl}}/api/public/scores?page=&limit=&userId=&name=",
"host": [
@@ -683,6 +683,38 @@
},
"response": []
},
{
"_type": "endpoint",
"name": "Get By Id",
"request": {
"description": "Get a score",
"url": {
"raw": "{{baseUrl}}/api/public/scores/:scoreId",
"host": [
"{{baseUrl}}"
],
"path": [
"api",
"public",
"scores",
":scoreId"
],
"query": [],
"variable": [
{
"key": "scoreId",
"value": "",
"description": "The unique langfuse identifier of a score"
}
]
},
"header": [],
"method": "GET",
"auth": null,
"body": null
},
"response": []
},
{
"_type": "endpoint",
"name": "Delete",
+46
View File
@@ -14,6 +14,7 @@
"@types/jest": "^29.5.12",
"@typescript-eslint/eslint-plugin": "^7.0.2",
"@typescript-eslint/parser": "^7.0.2",
"dotenv-cli": "^7.3.0",
"eslint": "^8.56.0",
"prettier": "^3.2.5"
}
@@ -926,6 +927,42 @@
"node": ">=6.0.0"
}
},
"node_modules/dotenv": {
"version": "16.4.5",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz",
"integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==",
"dev": true,
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dotenv-cli": {
"version": "7.3.0",
"resolved": "https://registry.npmjs.org/dotenv-cli/-/dotenv-cli-7.3.0.tgz",
"integrity": "sha512-314CA4TyK34YEJ6ntBf80eUY+t1XaFLyem1k9P0sX1gn30qThZ5qZr/ZwE318gEnzyYP9yj9HJk6SqwE0upkfw==",
"dev": true,
"dependencies": {
"cross-spawn": "^7.0.3",
"dotenv": "^16.3.0",
"dotenv-expand": "^10.0.0",
"minimist": "^1.2.6"
},
"bin": {
"dotenv": "cli.js"
}
},
"node_modules/dotenv-expand": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz",
"integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==",
"dev": true,
"engines": {
"node": ">=12"
}
},
"node_modules/escape-string-regexp": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
@@ -1601,6 +1638,15 @@
"node": "*"
}
},
"node_modules/minimist": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
"dev": true,
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+5 -3
View File
@@ -2,10 +2,11 @@
"name": "langfuse",
"version": "2.7.0",
"scripts": {
"infra:dev:up": "docker-compose -f ../docker-compose.dev.yml up -d",
"infra:dev:down": "docker-compose -f ../docker-compose.dev.yml down",
"infra:dev:up": "docker-compose -f ./docker-compose.dev.yml up -d",
"infra:dev:down": "docker-compose -f ./docker-compose.dev.yml down",
"dev": "npm run dev --prefix worker/ & npm run dev --prefix web/",
"postinstall": "cd worker && npm install && cd ../web && npm install"
"postinstall": "cd worker && npm install && cd ../web && npm install",
"dx": "dotenv -e .env -- npm i && npm run infra:dev:up && cd web && npm run db:reset && npm run db:seed:examples && cd .. && npm run dev"
},
"author": "",
"license": "MIT",
@@ -14,6 +15,7 @@
"@types/jest": "^29.5.12",
"@typescript-eslint/eslint-plugin": "^7.0.2",
"@typescript-eslint/parser": "^7.0.2",
"dotenv-cli": "^7.3.0",
"eslint": "^8.56.0",
"prettier": "^3.2.5"
}
+65 -8
View File
@@ -16,7 +16,7 @@ info:
paths:
/api/public/dataset-items:
post:
description: Create a dataset item, upserts on id
description: Create a dataset item
operationId: datasetItems_create
tags:
- DatasetItems
@@ -63,7 +63,7 @@ paths:
$ref: '#/components/schemas/CreateDatasetItemRequest'
/api/public/dataset-items/{id}:
get:
description: Get a specific dataset item
description: Get a dataset item
operationId: datasetItems_get
tags:
- DatasetItems
@@ -336,7 +336,7 @@ paths:
description: ''
/api/public/ingestion:
post:
description: Ingest multiple events to Langfuse
description: Batched ingestion for Langfuse Tracing
operationId: ingestion_batch
tags:
- Ingestion
@@ -464,7 +464,7 @@ paths:
security: *ref_0
/api/public/observations/{observationId}:
get:
description: Get a specific observation
description: Get a observation
operationId: observations_get
tags:
- Observations
@@ -594,6 +594,7 @@ paths:
security: *ref_0
/api/public/projects:
get:
description: Get Project associated with API key
operationId: projects_get
tags:
- Projects
@@ -633,7 +634,7 @@ paths:
security: *ref_0
/api/public/prompts:
get:
description: Get a specific prompt
description: Get a prompt
operationId: prompts_get
tags:
- Prompts
@@ -683,7 +684,7 @@ paths:
schema: {}
security: *ref_0
post:
description: Create a specific prompt
description: Create a prompt
operationId: prompts_create
tags:
- Prompts
@@ -729,7 +730,7 @@ paths:
$ref: '#/components/schemas/CreatePromptRequest'
/api/public/scores:
post:
description: Add a score to the database, upserts on id
description: Create a score
operationId: score_create
tags:
- Score
@@ -774,7 +775,7 @@ paths:
schema:
$ref: '#/components/schemas/CreateScoreRequest'
get:
description: Get scores
description: Get a list of scores
operationId: score_get
tags:
- Score
@@ -837,6 +838,51 @@ paths:
schema: {}
security: *ref_0
/api/public/scores/{scoreId}:
get:
description: Get a score
operationId: score_get-by-id
tags:
- Score
parameters:
- name: scoreId
in: path
description: The unique langfuse identifier of a score
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/Score'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security: *ref_0
delete:
description: Delete a score
operationId: score_delete
@@ -1283,16 +1329,20 @@ components:
Usage:
title: Usage
type: object
description: Standard interface for usage and cost
properties:
input:
type: integer
nullable: true
description: Number of input units (e.g. tokens)
output:
type: integer
nullable: true
description: Number of output units (e.g. tokens)
total:
type: integer
nullable: true
description: Defaults to input+output if not set
unit:
$ref: '#/components/schemas/ModelUsageUnit'
nullable: true
@@ -1300,14 +1350,17 @@ components:
type: number
format: double
nullable: true
description: USD input cost
outputCost:
type: number
format: double
nullable: true
description: USD output cost
totalCost:
type: number
format: double
nullable: true
description: USD total cost, defaults to input+output
Score:
title: Score
type: object
@@ -1458,6 +1511,7 @@ components:
- MILLISECONDS
- SECONDS
- IMAGES
description: Unit of usage in Langfuse
ObservationLevel:
title: ObservationLevel
type: string
@@ -1497,6 +1551,7 @@ components:
id:
type: string
nullable: true
description: Dataset items are upserted on their id
required:
- datasetName
- input
@@ -1664,6 +1719,7 @@ components:
OpenAIUsage:
title: OpenAIUsage
type: object
description: Usage interface of OpenAI for improved compatibility.
properties:
promptTokens:
type: integer
@@ -2193,6 +2249,7 @@ components:
type: string
isActive:
type: boolean
description: Should the prompt be promoted to production immediately?
prompt:
type: string
config:
+138 -138
View File
@@ -1,23 +1,23 @@
{
"name": "langfuse-core",
"version": "2.7.2",
"version": "2.9.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "langfuse-core",
"version": "2.7.2",
"version": "2.9.0",
"hasInstallScript": true,
"dependencies": {
"@anthropic-ai/tokenizer": "^0.0.4",
"@aws-sdk/client-s3": "^3.507.0",
"@aws-sdk/lib-storage": "^3.511.0",
"@aws-sdk/lib-storage": "^3.515.0",
"@aws-sdk/s3-request-presigner": "^3.507.0",
"@headlessui/react": "^1.7.18",
"@heroicons/react": "^2.1.1",
"@hookform/resolvers": "^3.3.4",
"@next-auth/prisma-adapter": "^1.0.7",
"@prisma/client": "^5.9.1",
"@prisma/client": "^5.10.2",
"@radix-ui/react-accordion": "^1.1.2",
"@radix-ui/react-alert-dialog": "^1.0.5",
"@radix-ui/react-avatar": "^1.0.4",
@@ -40,8 +40,8 @@
"@radix-ui/react-tooltip": "^1.0.7",
"@react-email/components": "^0.0.14",
"@react-email/render": "^0.0.12",
"@sentry/nextjs": "^7.101.1",
"@sentry/profiling-node": "^7.101.1",
"@sentry/nextjs": "^7.102.1",
"@sentry/profiling-node": "^7.102.1",
"@sentry/types": "^7.88.0",
"@t3-oss/env-nextjs": "^0.8.0",
"@tailwindcss/forms": "^0.5.7",
@@ -116,9 +116,9 @@
"postcss": "^8.4.35",
"prettier": "^3.2.5",
"prettier-plugin-tailwindcss": "^0.5.11",
"prisma": "^5.9.1",
"prisma": "^5.10.2",
"prisma-erd-generator": "^1.11.2",
"release-it": "^17.0.5",
"release-it": "^17.1.1",
"tailwindcss": "^3.4.1",
"ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0",
@@ -683,9 +683,9 @@
}
},
"node_modules/@aws-sdk/lib-storage": {
"version": "3.511.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.511.0.tgz",
"integrity": "sha512-inEbSyqzGxiQs8aEnkGdxw9ZDn370mRHOdE1TB/GvVe9buQVyZ2hQvOY5WBVOaIGDIxGpuUzVvr4o89XreU19w==",
"version": "3.515.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.515.0.tgz",
"integrity": "sha512-/7z/3KnMs1ODNS9c8Skj/DFTsy6/v7n17clh1IGOcTYhhioCMA3MIzIZecWFeLjPYcUSkNQHIIjKFQt1nhZkwA==",
"dependencies": {
"@smithy/abort-controller": "^2.1.1",
"@smithy/middleware-endpoint": "^2.4.1",
@@ -3459,9 +3459,9 @@
}
},
"node_modules/@prisma/client": {
"version": "5.9.1",
"resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.9.1.tgz",
"integrity": "sha512-caSOnG4kxcSkhqC/2ShV7rEoWwd3XrftokxJqOCMVvia4NYV/TPtJlS9C2os3Igxw/Qyxumj9GBQzcStzECvtQ==",
"version": "5.10.2",
"resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.10.2.tgz",
"integrity": "sha512-ef49hzB2yJZCvM5gFHMxSFL9KYrIP9udpT5rYo0CsHD4P9IKj473MbhU1gjKKftiwWBTIyrt9jukprzZXazyag==",
"hasInstallScript": true,
"engines": {
"node": ">=16.13"
@@ -3482,45 +3482,45 @@
"dev": true
},
"node_modules/@prisma/engines": {
"version": "5.9.1",
"resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.9.1.tgz",
"integrity": "sha512-gkdXmjxQ5jktxWNdDA5aZZ6R8rH74JkoKq6LD5mACSvxd2vbqWeWIOV0Py5wFC8vofOYShbt6XUeCIUmrOzOnQ==",
"version": "5.10.2",
"resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.10.2.tgz",
"integrity": "sha512-HkSJvix6PW8YqEEt3zHfCYYJY69CXsNdhU+wna+4Y7EZ+AwzeupMnUThmvaDA7uqswiHkgm5/SZ6/4CStjaGmw==",
"devOptional": true,
"hasInstallScript": true,
"dependencies": {
"@prisma/debug": "5.9.1",
"@prisma/engines-version": "5.9.0-32.23fdc5965b1e05fc54e5f26ed3de66776b93de64",
"@prisma/fetch-engine": "5.9.1",
"@prisma/get-platform": "5.9.1"
"@prisma/debug": "5.10.2",
"@prisma/engines-version": "5.10.0-34.5a9203d0590c951969e85a7d07215503f4672eb9",
"@prisma/fetch-engine": "5.10.2",
"@prisma/get-platform": "5.10.2"
}
},
"node_modules/@prisma/engines-version": {
"version": "5.9.0-32.23fdc5965b1e05fc54e5f26ed3de66776b93de64",
"resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.9.0-32.23fdc5965b1e05fc54e5f26ed3de66776b93de64.tgz",
"integrity": "sha512-HFl7275yF0FWbdcNvcSRbbu9JCBSLMcurYwvWc8WGDnpu7APxQo2ONtZrUggU3WxLxUJ2uBX+0GOFIcJeVeOOQ==",
"version": "5.10.0-34.5a9203d0590c951969e85a7d07215503f4672eb9",
"resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.10.0-34.5a9203d0590c951969e85a7d07215503f4672eb9.tgz",
"integrity": "sha512-uCy/++3Jx/O3ufM+qv2H1L4tOemTNqcP/gyEVOlZqTpBvYJUe0tWtW0y3o2Ueq04mll4aM5X3f6ugQftOSLdFQ==",
"devOptional": true
},
"node_modules/@prisma/engines/node_modules/@prisma/debug": {
"version": "5.9.1",
"resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.9.1.tgz",
"integrity": "sha512-yAHFSFCg8KVoL0oRUno3m60GAjsUKYUDkQ+9BA2X2JfVR3kRVSJFc/GpQ2fSORi4pSHZR9orfM4UC9OVXIFFTA==",
"version": "5.10.2",
"resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.10.2.tgz",
"integrity": "sha512-bkBOmH9dpEBbMKFJj8V+Zp8IZHIBjy3fSyhLhxj4FmKGb/UBSt9doyfA6k1UeUREsMJft7xgPYBbHSOYBr8XCA==",
"devOptional": true
},
"node_modules/@prisma/fetch-engine": {
"version": "5.9.1",
"resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.9.1.tgz",
"integrity": "sha512-l0goQOMcNVOJs1kAcwqpKq3ylvkD9F04Ioe1oJoCqmz05mw22bNAKKGWuDd3zTUoUZr97va0c/UfLNru+PDmNA==",
"version": "5.10.2",
"resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.10.2.tgz",
"integrity": "sha512-dSmXcqSt6DpTmMaLQ9K8ZKzVAMH3qwGCmYEZr/uVnzVhxRJ1EbT/w2MMwIdBNq1zT69Rvh0h75WMIi0mrIw7Hg==",
"devOptional": true,
"dependencies": {
"@prisma/debug": "5.9.1",
"@prisma/engines-version": "5.9.0-32.23fdc5965b1e05fc54e5f26ed3de66776b93de64",
"@prisma/get-platform": "5.9.1"
"@prisma/debug": "5.10.2",
"@prisma/engines-version": "5.10.0-34.5a9203d0590c951969e85a7d07215503f4672eb9",
"@prisma/get-platform": "5.10.2"
}
},
"node_modules/@prisma/fetch-engine/node_modules/@prisma/debug": {
"version": "5.9.1",
"resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.9.1.tgz",
"integrity": "sha512-yAHFSFCg8KVoL0oRUno3m60GAjsUKYUDkQ+9BA2X2JfVR3kRVSJFc/GpQ2fSORi4pSHZR9orfM4UC9OVXIFFTA==",
"version": "5.10.2",
"resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.10.2.tgz",
"integrity": "sha512-bkBOmH9dpEBbMKFJj8V+Zp8IZHIBjy3fSyhLhxj4FmKGb/UBSt9doyfA6k1UeUREsMJft7xgPYBbHSOYBr8XCA==",
"devOptional": true
},
"node_modules/@prisma/generator-helper": {
@@ -3533,18 +3533,18 @@
}
},
"node_modules/@prisma/get-platform": {
"version": "5.9.1",
"resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.9.1.tgz",
"integrity": "sha512-6OQsNxTyhvG+T2Ksr8FPFpuPeL4r9u0JF0OZHUBI/Uy9SS43sPyAIutt4ZEAyqWQt104ERh70EZedkHZKsnNbg==",
"version": "5.10.2",
"resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.10.2.tgz",
"integrity": "sha512-nqXP6vHiY2PIsebBAuDeWiUYg8h8mfjBckHh6Jezuwej0QJNnjDiOq30uesmg+JXxGk99nqyG3B7wpcOODzXvg==",
"devOptional": true,
"dependencies": {
"@prisma/debug": "5.9.1"
"@prisma/debug": "5.10.2"
}
},
"node_modules/@prisma/get-platform/node_modules/@prisma/debug": {
"version": "5.9.1",
"resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.9.1.tgz",
"integrity": "sha512-yAHFSFCg8KVoL0oRUno3m60GAjsUKYUDkQ+9BA2X2JfVR3kRVSJFc/GpQ2fSORi4pSHZR9orfM4UC9OVXIFFTA==",
"version": "5.10.2",
"resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.10.2.tgz",
"integrity": "sha512-bkBOmH9dpEBbMKFJj8V+Zp8IZHIBjy3fSyhLhxj4FmKGb/UBSt9doyfA6k1UeUREsMJft7xgPYBbHSOYBr8XCA==",
"devOptional": true
},
"node_modules/@prisma/internals": {
@@ -5291,57 +5291,57 @@
}
},
"node_modules/@sentry-internal/feedback": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-7.101.1.tgz",
"integrity": "sha512-fOKDMVvLX+FuJHJszKBvRg1m7+fd4hchqRnZ9DDfitT6P5Ppl0gbEt/LStqu8Wq5M0tna+hpdwHlVEt7gZVKzw==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-7.102.1.tgz",
"integrity": "sha512-vY4hpLLMNLjICtWiizc7KeGbWOTUMGrF7C+9dPCztZww3CLgzWy9A7DvPj5hodRiYzpdRnAMl8yQnMFbYXh7bA==",
"dependencies": {
"@sentry/core": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1"
"@sentry/core": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@sentry-internal/replay-canvas": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-7.101.1.tgz",
"integrity": "sha512-09l6nD+lxWvwkpXLlIZuzj/z79Llbo6mcH33TJvxrUTjAqSGF/i3Pd5bTLWro9atippOyQgIV/yTGG4Bc5FhyQ==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-7.102.1.tgz",
"integrity": "sha512-GUX4RWI10uRjdjeyvCLtAAhWRVqnAnG6+yNxWfqUQ3qMA7B7XxG43KT2UhSnulmErNzODQ6hA68rGPwwYeRIww==",
"dependencies": {
"@sentry/core": "7.101.1",
"@sentry/replay": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1"
"@sentry/core": "7.102.1",
"@sentry/replay": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@sentry-internal/tracing": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.101.1.tgz",
"integrity": "sha512-ihjWG8x4x0ozx6t+EHoXLKbsPrgzYLCpeBLWyS+M6n3hn6cmHM76c8nZw3ldhUQi5UYL3LFC/JZ50b4oSxtlrg==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.102.1.tgz",
"integrity": "sha512-RkFlFyAC0fQOvBbBqnq0CLmFW5m3JJz9pKbZd5vXPraWAlniKSb1bC/4DF9SlNx0FN1LWG+IU3ISdpzwwTeAGg==",
"dependencies": {
"@sentry/core": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1"
"@sentry/core": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@sentry/browser": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-7.101.1.tgz",
"integrity": "sha512-+rIFoWPdO29AHVYsAwq8QEl2Ihv17Xh9Bt2aPFvLTGDA0caHjnx98g2jSOvLIOah6HI7Nwp3Njg2zBEzDtHkNw==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-7.102.1.tgz",
"integrity": "sha512-7BOfPBiM7Kp6q/iy0JIbsBTxIASV+zWXByqqjuEMWGj3X2u4oRIfm3gv4erPU/l+CORQUVQZLSPGoIoM1gbB/A==",
"dependencies": {
"@sentry-internal/feedback": "7.101.1",
"@sentry-internal/replay-canvas": "7.101.1",
"@sentry-internal/tracing": "7.101.1",
"@sentry/core": "7.101.1",
"@sentry/replay": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1"
"@sentry-internal/feedback": "7.102.1",
"@sentry-internal/replay-canvas": "7.102.1",
"@sentry-internal/tracing": "7.102.1",
"@sentry/core": "7.102.1",
"@sentry/replay": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1"
},
"engines": {
"node": ">=8"
@@ -5368,25 +5368,25 @@
}
},
"node_modules/@sentry/core": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.101.1.tgz",
"integrity": "sha512-XSmXXeYT1d4O14eDF3OXPJFUgaN2qYEeIGUztqPX9nBs9/ij8y/kZOayFqlIMnfGvjOUM+63sy/2xDBOpFn6ug==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.102.1.tgz",
"integrity": "sha512-QjY+LSP3du3J/C8x/FfEbRxgZgsWd0jfTJ4P7s9f219I1csK4OeBMC3UA1HwEa0pY/9OF6H/egW2CjOcMM5Pdg==",
"dependencies": {
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1"
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@sentry/integrations": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/integrations/-/integrations-7.101.1.tgz",
"integrity": "sha512-0kk6773Lg2Pa1cUmK1VtxaLLAmpIXcT3qYPlYxRXZQJEUpGcjZZ704V7i8SFLhJSsHkXrL/sAGSyM1p+NDF+QA==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/integrations/-/integrations-7.102.1.tgz",
"integrity": "sha512-Its3Ru6xCAqpaLE3cTxW/b91js2SIFoXa8LWtQDJ7tmTdwPAbT8Pij1F4bOhhaqLYbjLtCXGl/NR2cffsiRLww==",
"dependencies": {
"@sentry/core": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1",
"@sentry/core": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1",
"localforage": "^1.8.1"
},
"engines": {
@@ -5394,18 +5394,18 @@
}
},
"node_modules/@sentry/nextjs": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/nextjs/-/nextjs-7.101.1.tgz",
"integrity": "sha512-qIT0jByxaAbbmX5Gl//M9+d+Pi1znk9aWoIGZyakpK+seGQ3D+UjrznxxmwvjcqQgVpbo0SJSq+3yQv7eAh2Tg==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/nextjs/-/nextjs-7.102.1.tgz",
"integrity": "sha512-gOI/GD7DWhc3WucyYnepl8Nu5jmpa1YfR6jWDzTkPE2CV9zKK9zulTdqk+Aig9ch62SAmJOGkgejm5k9PE2XzQ==",
"dependencies": {
"@rollup/plugin-commonjs": "24.0.0",
"@sentry/core": "7.101.1",
"@sentry/integrations": "7.101.1",
"@sentry/node": "7.101.1",
"@sentry/react": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1",
"@sentry/vercel-edge": "7.101.1",
"@sentry/core": "7.102.1",
"@sentry/integrations": "7.102.1",
"@sentry/node": "7.102.1",
"@sentry/react": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1",
"@sentry/vercel-edge": "7.102.1",
"@sentry/webpack-plugin": "1.21.0",
"chalk": "3.0.0",
"resolve": "1.22.8",
@@ -5427,23 +5427,23 @@
}
},
"node_modules/@sentry/node": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.101.1.tgz",
"integrity": "sha512-iXSxUT6Zbt/KUY0+fRcW5II6Tgp2zdTfhBW+fQuDt/UUZt7Ypvb+6n4U2oom3LJfttmD7mdjQuT4+vsNImDjTQ==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.102.1.tgz",
"integrity": "sha512-mb3vmM3SGuCruckPiv/Vafeh89UQavTfpPFoU6Jwe6dSpQ39BO8fO8k8Zev+/nP6r/FKLtX17mJobErHECXsYw==",
"dependencies": {
"@sentry-internal/tracing": "7.101.1",
"@sentry/core": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1"
"@sentry-internal/tracing": "7.102.1",
"@sentry/core": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@sentry/profiling-node": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/profiling-node/-/profiling-node-7.101.1.tgz",
"integrity": "sha512-buM+HZZW7jf3mqJgByVGPwytvmHcbjxRT8CkjO5UDO7stRTsq3iT/hK7qt0bQEDQwhmKeVCvfqryg2nKkPq3RA==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/profiling-node/-/profiling-node-7.102.1.tgz",
"integrity": "sha512-eqOUdu04eI4ODqeh/nHvC/mdwm3tWkqm02anR2ITEjKVJxHliHH6+jr+3M2X56e1hIxOibtL+JrR89Du9HEl9w==",
"hasInstallScript": true,
"dependencies": {
"detect-libc": "^2.0.2",
@@ -5457,14 +5457,14 @@
}
},
"node_modules/@sentry/react": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/react/-/react-7.101.1.tgz",
"integrity": "sha512-CwaBXntX2e3XHZQZVuv/tcfm5H+UHcS6aVChGfUiBHIBi2JpAqdnLdQIFGTkE8BSnKyolKgIsnvIU3BQ//QTig==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/react/-/react-7.102.1.tgz",
"integrity": "sha512-X4j2DgbktlEifnd21YJKCayAmff5hnaS+9MNz9OonEwD0ARi0ks7bo0wtWHMjPK20992MO+JwczVg/1BXJYDdQ==",
"dependencies": {
"@sentry/browser": "7.101.1",
"@sentry/core": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1",
"@sentry/browser": "7.102.1",
"@sentry/core": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1",
"hoist-non-react-statics": "^3.3.2"
},
"engines": {
@@ -5475,47 +5475,47 @@
}
},
"node_modules/@sentry/replay": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/replay/-/replay-7.101.1.tgz",
"integrity": "sha512-l4jmj2Rf/myzk3TA83PdMiomassG8okdBh1b2Hp1+ycBRVZFDmsR81gKPvnefSXwGwGNGKEmp6Q2bdGzekpp3Q==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/replay/-/replay-7.102.1.tgz",
"integrity": "sha512-HR/j9dGIvbrId8fh8mQlODx7JrhRmawEd9e9P3laPtogWCg/5TI+XPb2VGSaXOX9VWtb/6Z2UjHsaGjgg6YcuA==",
"dependencies": {
"@sentry-internal/tracing": "7.101.1",
"@sentry/core": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1"
"@sentry-internal/tracing": "7.102.1",
"@sentry/core": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@sentry/types": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.101.1.tgz",
"integrity": "sha512-bwtkQvrCZ6JGc7vqX7TEAKBgkbQFORt84FFS3JQQb8G3efTt9fZd2ReY4buteKQdlALl8h1QWVngTLmI+kyUuw==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.102.1.tgz",
"integrity": "sha512-htKorf3t/D0XYtM7foTcmG+rM47rDP6XdbvCcX5gBCuCYlzpM1vqCt2rl3FLktZC6TaIpFRJw1TLfx6m+x5jdA==",
"engines": {
"node": ">=8"
}
},
"node_modules/@sentry/utils": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.101.1.tgz",
"integrity": "sha512-Nrg0nrEI3nrOCd9SLJ/WGzxS5KMQE4cryLOvrDcHJRWpsSyGBF1hLLerk84Nsw/0myMsn7zTYU+xoq7idNsX5A==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.102.1.tgz",
"integrity": "sha512-+8WcFjHVV/HROXSAwMuUzveElBFC43EiTG7SNEBNgOUeQzQVTmbUZXyTVgLrUmtoWqvnIxCacoLxtZo1o67kdg==",
"dependencies": {
"@sentry/types": "7.101.1"
"@sentry/types": "7.102.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@sentry/vercel-edge": {
"version": "7.101.1",
"resolved": "https://registry.npmjs.org/@sentry/vercel-edge/-/vercel-edge-7.101.1.tgz",
"integrity": "sha512-xs6Xq910CR9rWHCYJEoEbfIfH60Oce+09NQTS5h3R/SOmrrAv4/mmZyblhunIS8TJqABeGRtEPKt1lOqO20O/A==",
"version": "7.102.1",
"resolved": "https://registry.npmjs.org/@sentry/vercel-edge/-/vercel-edge-7.102.1.tgz",
"integrity": "sha512-iB6KCSxrvO172VjfQHGiYpyXPKNbx6Cz01GA1YDByRiUSgSpAq0qAFRY4lvd736w5KX4s9hen+XmZ7Gmqj8Pag==",
"dependencies": {
"@sentry-internal/tracing": "7.101.1",
"@sentry/core": "7.101.1",
"@sentry/types": "7.101.1",
"@sentry/utils": "7.101.1"
"@sentry-internal/tracing": "7.102.1",
"@sentry/core": "7.102.1",
"@sentry/types": "7.102.1",
"@sentry/utils": "7.102.1"
},
"engines": {
"node": ">=8"
@@ -16299,13 +16299,13 @@
"dev": true
},
"node_modules/prisma": {
"version": "5.9.1",
"resolved": "https://registry.npmjs.org/prisma/-/prisma-5.9.1.tgz",
"integrity": "sha512-Hy/8KJZz0ELtkw4FnG9MS9rNWlXcJhf98Z2QMqi0QiVMoS8PzsBkpla0/Y5hTlob8F3HeECYphBjqmBxrluUrQ==",
"version": "5.10.2",
"resolved": "https://registry.npmjs.org/prisma/-/prisma-5.10.2.tgz",
"integrity": "sha512-hqb/JMz9/kymRE25pMWCxkdyhbnIWrq+h7S6WysJpdnCvhstbJSNP/S6mScEcqiB8Qv2F+0R3yG+osRaWqZacQ==",
"devOptional": true,
"hasInstallScript": true,
"dependencies": {
"@prisma/engines": "5.9.1"
"@prisma/engines": "5.10.2"
},
"bin": {
"prisma": "build/index.js"
@@ -17237,9 +17237,9 @@
}
},
"node_modules/release-it": {
"version": "17.0.5",
"resolved": "https://registry.npmjs.org/release-it/-/release-it-17.0.5.tgz",
"integrity": "sha512-97FcBu/2IjPd4qMvliZSnavpCO0+BWi8lZr+BgVJTXs6ihX8HLPucYR5RsgUUcZoGKj3beO8cunrsMuvEWTzGQ==",
"version": "17.1.1",
"resolved": "https://registry.npmjs.org/release-it/-/release-it-17.1.1.tgz",
"integrity": "sha512-b+4Tu2eb5f2wIdIe5E9hre0evbMQrXp/kRq0natHsHYJVqu1Bd4/h2a+swFi0faGmC3cJdB16uYR6LscG9SchQ==",
"dev": true,
"funding": [
{
+12 -9
View File
@@ -1,6 +1,6 @@
{
"name": "langfuse-core",
"version": "2.7.2",
"version": "2.9.0",
"private": true,
"scripts": {
"prebuild": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
@@ -16,12 +16,15 @@
"test": "dotenv -e ../.env -- jest --runInBand",
"test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
"test:e2e": "dotenv -e ../.env -- playwright test",
"db:migrate": "DISABLE_ERD=false dotenv -e ../.env -- npx prisma migrate dev",
"db:migrate": "DISABLE_ERD=false dotenv -e ../.env -- npx prisma migrate dev && npm run db:worker",
"db:reset": "dotenv -e ../.env npx -- prisma migrate reset",
"db:deploy": "dotenv -e ../.env npx -- prisma migrate deploy",
"db:seed": "dotenv -e ../.env -- npx prisma db seed",
"db:seed:examples": "dotenv -e ../.env -- npx prisma db seed -- --environment examples",
"db:seed:load": "dotenv -e ../.env -- npx prisma db seed -- --environment load",
"release": "dotenv -e ../.env -- release-it",
"models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
"models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts",
"db:worker": "cp -r prisma/generated ../worker"
},
"prisma": {
"seed": "ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options {\"module\":\"CommonJS\"} prisma/seed.ts"
@@ -29,13 +32,13 @@
"dependencies": {
"@anthropic-ai/tokenizer": "^0.0.4",
"@aws-sdk/client-s3": "^3.507.0",
"@aws-sdk/lib-storage": "^3.511.0",
"@aws-sdk/lib-storage": "^3.515.0",
"@aws-sdk/s3-request-presigner": "^3.507.0",
"@headlessui/react": "^1.7.18",
"@heroicons/react": "^2.1.1",
"@hookform/resolvers": "^3.3.4",
"@next-auth/prisma-adapter": "^1.0.7",
"@prisma/client": "^5.9.1",
"@prisma/client": "^5.10.2",
"@radix-ui/react-accordion": "^1.1.2",
"@radix-ui/react-alert-dialog": "^1.0.5",
"@radix-ui/react-avatar": "^1.0.4",
@@ -58,8 +61,8 @@
"@radix-ui/react-tooltip": "^1.0.7",
"@react-email/components": "^0.0.14",
"@react-email/render": "^0.0.12",
"@sentry/nextjs": "^7.101.1",
"@sentry/profiling-node": "^7.101.1",
"@sentry/nextjs": "^7.102.1",
"@sentry/profiling-node": "^7.102.1",
"@sentry/types": "^7.88.0",
"@t3-oss/env-nextjs": "^0.8.0",
"@tailwindcss/forms": "^0.5.7",
@@ -134,9 +137,9 @@
"postcss": "^8.4.35",
"prettier": "^3.2.5",
"prettier-plugin-tailwindcss": "^0.5.11",
"prisma": "^5.9.1",
"prisma": "^5.10.2",
"prisma-erd-generator": "^1.11.2",
"release-it": "^17.0.5",
"release-it": "^17.1.1",
"tailwindcss": "^3.4.1",
"ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0",
+1 -1
View File
@@ -225,7 +225,7 @@ export type ObservationView = {
calculated_input_cost: string | null;
calculated_output_cost: string | null;
calculated_total_cost: string | null;
latency: string | null;
latency: number | null;
};
export type Pricing = {
id: string;
@@ -0,0 +1,2 @@
-- CreateIndex
CREATE INDEX CONCURRENTLY "observations_project_id_start_time_type_idx" ON "observations"("project_id", "start_time", "type");
@@ -0,0 +1,2 @@
-- CreateIndex
CREATE INDEX CONCURRENTLY "models_project_id_model_name_start_date_unit_idx" ON "models"("project_id", "model_name", "start_date", "unit");
@@ -0,0 +1,2 @@
-- CreateIndex
CREATE INDEX CONCURRENTLY "observations_project_id_internal_model_start_time_unit_idx" ON "observations"("project_id", "internal_model", "start_time", "unit");
@@ -0,0 +1,2 @@
-- CreateIndex
CREATE INDEX CONCURRENTLY "observations_trace_id_project_id_start_time_idx" ON "observations"("trace_id", "project_id", "start_time");
@@ -0,0 +1,2 @@
-- CreateIndex
CREATE INDEX CONCURRENTLY "observations_trace_id_project_id_type_start_time_idx" ON "observations"("trace_id", "project_id", "type", "start_time");
@@ -0,0 +1,50 @@
CREATE OR REPLACE VIEW "observations_view" AS
SELECT
o.*,
m.id AS "model_id",
m.start_date AS "model_start_date",
m.input_price,
m.output_price,
m.total_price,
m.tokenizer_config AS "tokenizer_config",
CASE
WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
o.prompt_tokens::decimal * m.input_price
ELSE
o.input_cost
END AS "calculated_input_cost",
CASE
WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
o.completion_tokens::decimal * m.output_price
ELSE
o.output_cost
END AS "calculated_output_cost",
CASE
WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
CASE
WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
m.total_price * o.total_tokens
ELSE
o.prompt_tokens::decimal * m.input_price +
o.completion_tokens::decimal * m.output_price
END
ELSE
o.total_cost
END AS "calculated_total_cost",
CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency"
FROM
observations o
LEFT JOIN LATERAL (
SELECT
models.*
FROM
models
WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
AND models.model_name = o.internal_model
AND (models.start_date < o.start_time OR models.start_date IS NULL)
AND o.unit::TEXT = models.unit
ORDER BY
models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
LIMIT 1
) m ON TRUE
@@ -0,0 +1,2 @@
-- CreateIndex
CREATE INDEX CONCURRENTLY IF NOT EXISTS "observations_prompt_id_idx" ON "observations"("prompt_id");
@@ -0,0 +1,54 @@
CREATE OR REPLACE VIEW "observations_view" AS
WITH model_ranked AS (
SELECT
*,
ROW_NUMBER() OVER (
PARTITION BY project_id, model_name, unit
ORDER BY project_id ASC, start_date DESC NULLS LAST
) AS rn
-- adds a new column to the models table ordering the rows by project_id, model_name, and unit
-- each query should take the first row within a partition
FROM
models
)
SELECT
o.*,
m.id AS "model_id",
m.start_date AS "model_start_date",
m.input_price,
m.output_price,
m.total_price,
m.tokenizer_config AS "tokenizer_config",
CASE
WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
o.prompt_tokens::decimal * m.input_price
ELSE
o.input_cost
END AS "calculated_input_cost",
CASE
WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
o.completion_tokens::decimal * m.output_price
ELSE
o.output_cost
END AS "calculated_output_cost",
CASE
WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
CASE
WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
m.total_price * o.total_tokens
ELSE
o.prompt_tokens::decimal * m.input_price +
o.completion_tokens::decimal * m.output_price
END
ELSE
o.total_cost
END AS "calculated_total_cost",
CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency"
FROM
observations o
LEFT JOIN model_ranked m ON
m.rn = 1 AND
(m.project_id = o.project_id OR m.project_id IS NULL) AND
m.model_name = o.internal_model AND
(m.start_date < o.start_time OR m.start_date IS NULL) AND
o.unit::TEXT = m.unit;
@@ -0,0 +1,59 @@
CREATE OR REPLACE VIEW "observations_view" AS
SELECT
o.*,
m.id AS "model_id",
m.start_date AS "model_start_date",
m.input_price,
m.output_price,
m.total_price,
m.tokenizer_config AS "tokenizer_config",
CASE
WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
o.prompt_tokens::decimal * m.input_price
ELSE
o.input_cost
END AS "calculated_input_cost",
CASE
WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
o.completion_tokens::decimal * m.output_price
ELSE
o.output_cost
END AS "calculated_output_cost",
CASE
WHEN o.input_cost IS NULL AND o.output_cost IS NULL AND o.total_cost IS NULL THEN
CASE
WHEN m.total_price IS NOT NULL AND o.total_tokens IS NOT NULL THEN
m.total_price * o.total_tokens
ELSE
o.prompt_tokens::decimal * m.input_price +
o.completion_tokens::decimal * m.output_price
END
ELSE
o.total_cost
END AS "calculated_total_cost",
CASE WHEN o.end_time IS NULL THEN NULL ELSE (EXTRACT(EPOCH FROM o."end_time") - EXTRACT(EPOCH FROM o."start_time"))::double precision END AS "latency"
FROM
observations o
LEFT JOIN LATERAL (
SELECT
models.*
FROM
models
WHERE (models.project_id = o.project_id OR models.project_id IS NULL)
AND models.model_name = o.internal_model
AND (models.start_date < o.start_time OR models.start_date IS NULL)
AND o.unit::TEXT = models.unit
ORDER BY
models.project_id ASC, -- in postgres, NULLs are sorted last when ordering ASC
models.start_date DESC NULLS LAST -- now, NULLs are sorted last when ordering DESC as well
LIMIT 1
) m ON TRUE
-- requirements:
-- 1. The view should return all columns from the observations table
-- 2. The view should match with only one model for each observation if:
-- a. The model has the same project_id as the observation, otherwise the model without project_id.
-- b. The model has the same model_name as the observation
-- c. The model has a start_date that is less than the observation start_time, otherwise the model without start_date
-- d. The model has the same unit as the observation
+8 -2
View File
@@ -261,6 +261,9 @@ model Observation {
prompt Prompt? @relation(fields: [promptId], onDelete: SetNull, references: [id])
@@unique([id, projectId])
@@index([projectId, internalModel, startTime, unit])
@@index([traceId, projectId, type, startTime])
@@index([traceId, projectId, startTime])
@@index([traceId])
@@index([type])
@@index(startTime)
@@ -268,6 +271,8 @@ model Observation {
@@index(projectId)
@@index(parentObservationId)
@@index(model)
@@index(promptId)
@@index([projectId, startTime, type])
@@map("observations")
}
@@ -461,6 +466,7 @@ model Model {
tokenizerConfig Json? @map("tokenizer_config")
@@unique([projectId, modelName, startDate, unit])
@@index([projectId, modelName, startDate, unit])
@@index([projectId, modelName])
@@map("models")
}
@@ -495,7 +501,7 @@ view ObservationView {
promptId String? @map("prompt_id")
// Model
// model fields
modelId String? @map("model_id")
inputPrice Decimal? @map("input_price")
outputPrice Decimal? @map("output_price")
@@ -505,7 +511,7 @@ view ObservationView {
calculatedInputCost Decimal? @map("calculated_input_cost")
calculatedOutputCost Decimal? @map("calculated_output_cost")
calculatedTotalCost Decimal? @map("calculated_total_cost")
latency Decimal? @map("latency")
latency Float? @map("latency")
@@map("observations_view")
}
+632 -433
View File
File diff suppressed because it is too large Load Diff
+13 -2
View File
@@ -2,7 +2,7 @@ import { prisma } from "@/src/server/db";
import { jsonSchema } from "@/src/utils/zod";
import * as Sentry from "@sentry/nextjs";
import { ProfilingIntegration } from "@sentry/profiling-node";
import type { TransactionEvent } from "@sentry/types";
import type { SamplingContext, TransactionEvent } from "@sentry/types";
if (process.env.NEXT_PUBLIC_SENTRY_DSN)
Sentry.init({
@@ -12,7 +12,18 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN)
// Set tracesSampleRate to 1.0 to capture 100%
// of transactions for performance monitoring.
// We recommend adjusting this value in production
tracesSampleRate: 0.3,
tracesSampleRate: 1.0,
tracesSampler: (samplingContext: SamplingContext) => {
if (
samplingContext.request &&
samplingContext.request.method === "POST" &&
samplingContext.request.url &&
samplingContext.request.url.includes("api/trpc")
) {
return 1.0;
}
return 0.3;
},
profilesSampleRate: 0.3, // Profiling sample rate is relative to tracesSampleRate
integrations: [
+79 -2
View File
@@ -320,6 +320,85 @@ describe("cost retrieval tests", () => {
expect(view?.calculatedTotalCost?.toString()).toBe("0.0124");
});
it(`should take old model for old observations`, async () => {
await prisma.model.create({
data: {
id: "model-0",
modelName: "gpt-3.5-turbo",
inputPrice: "0.0000000",
outputPrice: "0.0000000",
totalPrice: "0.1",
matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
projectId: null,
startDate: null,
tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
unit: ModelUsageUnit.Tokens,
},
});
await prisma.model.create({
data: {
id: "model-1",
modelName: "gpt-3.5-turbo",
inputPrice: "0.0000010",
outputPrice: "0.0000020",
totalPrice: "0.1",
matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
projectId: null,
startDate: new Date("2023-12-01"),
tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
unit: ModelUsageUnit.Tokens,
},
});
await prisma.model.create({
data: {
id: "model-2",
modelName: "gpt-3.5-turbo",
inputPrice: "0.0000020",
outputPrice: "0.0000040",
totalPrice: undefined,
matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
startDate: new Date("2023-12-02"),
tokenizerConfig: { tokensPerMessage: 3, tokensPerName: 1 },
unit: ModelUsageUnit.Tokens,
},
});
const dbTrace = await prisma.trace.create({
data: {
name: "trace-name",
project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
},
});
await prisma.observation.create({
data: {
traceId: dbTrace.id,
type: "GENERATION",
project: { connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" } },
model: "gpt-3.5-turbo",
internalModel: "gpt-3.5-turbo",
startTime: new Date("2023-01-01T00:00:00.000Z"),
unit: ModelUsageUnit.Tokens,
promptTokens: 200,
completionTokens: 3000,
totalTokens: undefined,
},
});
const view = await prisma.observationView.findFirst({
where: { traceId: dbTrace.id },
});
console.log(view);
// calculated cost fields
expect(view?.modelId).toBe("model-0");
expect(view?.calculatedInputCost?.toString()).toBe("0");
expect(view?.calculatedOutputCost?.toString()).toBe("0");
expect(view?.calculatedTotalCost?.toString()).toBe("0");
});
it(`should prioritize own models`, async () => {
await pruneDatabase();
await prisma.model.create({
@@ -385,8 +464,6 @@ describe("cost retrieval tests", () => {
});
it(`should prioritize old model if the latest model is not own one`, async () => {
await pruneDatabase();
await prisma.model.create({
data: {
id: "model-1",
+43
View File
@@ -1763,4 +1763,47 @@ IB Home / . . . / News / News about the IB / Why ChatGPT is an o
expect(dbScore).toBeNull();
});
it("should error on wrong input", async () => {
const traceId = "trace_id";
const bearerAuth = "Bearer pk-lf-1234567890";
const scoreId = "score_id";
const scoreEventId = "score_event_id";
const scoreName = "score-name";
const scoreValue = 100.5;
const response = await makeAPICall(
"POST",
"/api/public/ingestion",
{
// sending data instead of batch
data: [
{
id: scoreEventId,
type: "score-create",
timestamp: new Date().toISOString(),
body: {
id: scoreId,
name: scoreName,
value: scoreValue,
traceId: traceId,
},
},
],
},
bearerAuth,
);
expect(response.status).toBe(400);
expect(await prisma.trace.count()).toBe(0);
const dbScore = await prisma.score.findUnique({
where: {
id: scoreId,
},
});
expect(dbScore).toBeNull();
});
});
+43
View File
@@ -270,4 +270,47 @@ describe("/api/public/scores API Endpoint", () => {
});
expect(deletedScore).toBeNull();
});
it("should GET a score", async () => {
await pruneDatabase();
const traceId = uuidv4();
await makeAPICall("POST", "/api/public/traces", {
id: traceId,
});
const generationId = uuidv4();
await makeAPICall("POST", "/api/public/generations", {
id: generationId,
});
const scoreId = uuidv4();
await makeAPICall("POST", "/api/public/scores", {
id: scoreId,
observationId: generationId,
name: "score-name",
value: 100.5,
traceId: traceId,
comment: "comment",
});
const getScore = await makeAPICall<{
id: string;
name: string;
value: number;
comment: string;
traceId: string;
observationId: string;
}>("GET", `/api/public/scores/${scoreId}`);
expect(getScore.status).toBe(200);
expect(getScore.body).toMatchObject({
id: scoreId,
name: "score-name",
value: 100.5,
comment: "comment",
traceId,
observationId: generationId,
});
});
});
+2 -2
View File
@@ -171,8 +171,8 @@ export function DatePickerWithRange({
{internalDateRange?.from ? (
internalDateRange.to ? (
<>
{format(internalDateRange.from, "LLL dd, yy : hh:mm")} -{" "}
{format(internalDateRange.to, "LLL dd, yy : hh:mm")}
{format(internalDateRange.from, "LLL dd, yy : HH:mm")} -{" "}
{format(internalDateRange.to, "LLL dd, yy : HH:mm")}
</>
) : (
format(internalDateRange.from, "LLL dd, y")
@@ -23,7 +23,7 @@ import {
} from "use-query-params";
import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";
import { observationsTableColsWithOptions } from "@/src/server/api/definitions/observationsTable";
import { formatInterval, utcDateOffsetByDays } from "@/src/utils/dates";
import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
import { JSONView } from "@/src/components/ui/code";
import { type LangfuseColumnDef } from "@/src/components/table/types";
@@ -36,6 +36,7 @@ import {
type ExportFileFormats,
} from "@/src/server/api/interfaces/exportTypes";
import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
import type Decimal from "decimal.js";
export type GenerationsTableRow = {
id: string;
@@ -44,11 +45,15 @@ export type GenerationsTableRow = {
level?: ObservationLevel;
statusMessage?: string;
endTime?: string;
timeToFirstToken?: string;
latency?: number;
name?: string;
model?: string;
input?: unknown;
output?: unknown;
inputCost?: Decimal;
outputCost?: Decimal;
totalCost?: Decimal;
traceName?: string;
metadata?: string;
scores: Score[];
@@ -57,6 +62,9 @@ export type GenerationsTableRow = {
completionTokens: number;
totalTokens: number;
};
promptId?: string;
promptName?: string;
promptVersion?: string;
};
export type GenerationsTableProps = {
@@ -119,6 +127,10 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
setIsExporting(true);
posthog.capture("generations:export", { file_format: fileFormat });
if (fileFormat === "OPENAI-JSONL")
alert(
"When exporting in OpenAI-JSONL, only generations that exactly match the `ChatML` format will be exported. For any questions, reach out to support.",
);
try {
const fileData = await directApi.generations.export.query({
projectId,
@@ -181,7 +193,7 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
{
accessorKey: "name",
id: "name",
header: "name",
header: "Name",
enableSorting: true,
},
{
@@ -213,6 +225,34 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
enableHiding: true,
enableSorting: true,
},
{
accessorKey: "endTime",
id: "endTime",
header: "End Time",
enableHiding: true,
enableSorting: true,
},
{
accessorKey: "timeToFirstToken",
id: "timeToFirstToken",
header: "Time to First Token",
enableHiding: true,
cell: ({ row }) => {
const startTime: string = row.getValue("startTime");
const timeToFirstToken: string | undefined =
row.getValue("timeToFirstToken");
if (!timeToFirstToken) {
return undefined;
}
const latencyInMs =
new Date(timeToFirstToken).getTime() - new Date(startTime).getTime();
const latencyInSeconds = latencyInMs / 1000;
return <span>{formatIntervalSeconds(latencyInSeconds)}</span>;
},
},
{
accessorKey: "scores",
id: "scores",
@@ -228,22 +268,71 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
id: "latency",
header: "Latency",
cell: ({ row }) => {
const value: number | undefined = row.getValue("latency");
return value !== undefined ? (
<span>{formatInterval(value)}</span>
const latency: number | undefined = row.getValue("latency");
return latency !== undefined ? (
<span>{formatIntervalSeconds(latency)}</span>
) : undefined;
},
enableHiding: true,
enableSorting: true,
},
{
accessorKey: "cost",
header: "Cost",
accessorKey: "timePerOutputToken",
id: "timePerOutputToken",
header: "Time per Output Token",
cell: ({ row }) => {
const value: number | undefined = row.getValue("cost");
const latency: number | undefined = row.getValue("latency");
const usage: {
promptTokens: number;
completionTokens: number;
totalTokens: number;
} = row.getValue("usage");
return latency !== undefined &&
(usage.completionTokens !== 0 || usage.totalTokens !== 0) ? (
<span>
{usage.completionTokens
? formatIntervalSeconds(latency / usage.completionTokens)
: formatIntervalSeconds(latency / usage.totalTokens)}
</span>
) : undefined;
},
defaultHidden: true,
enableHiding: true,
},
{
accessorKey: "inputCost",
header: "Input Cost",
cell: ({ row }) => {
const value: Decimal | undefined = row.getValue("inputCost");
return value !== undefined ? (
<span>{usdFormatter(value)}</span>
<span>{usdFormatter(value.toNumber())}</span>
) : undefined;
},
enableHiding: true,
defaultHidden: true,
},
{
accessorKey: "outputCost",
header: "Output Cost",
cell: ({ row }) => {
const value: Decimal | undefined = row.getValue("outputCost");
return value !== undefined ? (
<span>{usdFormatter(value.toNumber())}</span>
) : undefined;
},
enableHiding: true,
defaultHidden: true,
},
{
accessorKey: "totalCost",
header: "Total Cost",
cell: ({ row }) => {
const value: Decimal | undefined = row.getValue("totalCost");
return value !== undefined ? (
<span>{usdFormatter(value.toNumber())}</span>
) : undefined;
},
enableHiding: true,
@@ -339,6 +428,28 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
enableHiding: true,
enableSorting: true,
},
{
accessorKey: "prompt",
id: "prompt",
header: "Prompt",
enableHiding: true,
enableSorting: true,
cell: ({ row }) => {
const promptName = row.original.promptName;
const promptVersion = row.original.promptVersion;
const value = `${promptName} (v${promptVersion})`;
return (
promptName &&
promptVersion && (
<TableLink
path={`/project/${projectId}/prompts/${encodeURIComponent(promptName)}?version=${promptVersion}`}
value={value}
truncateAt={40}
/>
)
);
},
},
];
const [columnVisibility, setColumnVisibility] =
useColumnVisibility<GenerationsTableRow>(
@@ -354,8 +465,12 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
traceName: generation.traceName,
startTime: generation.startTime.toLocaleString(),
endTime: generation.endTime?.toLocaleString() ?? undefined,
latency: generation.latency === null ? undefined : generation.latency,
cost: generation.calculatedTotalCost,
timeToFirstToken:
generation.completionStartTime?.toLocaleString() ?? undefined,
latency: generation.latency ?? undefined,
totalCost: generation.calculatedTotalCost ?? undefined,
inputCost: generation.calculatedInputCost ?? undefined,
outputCost: generation.calculatedOutputCost ?? undefined,
name: generation.name ?? undefined,
version: generation.version ?? "",
model: generation.model ?? "",
@@ -372,6 +487,9 @@ export default function GenerationsTable({ projectId }: GenerationsTableProps) {
completionTokens: generation.completionTokens,
totalTokens: generation.totalTokens,
},
promptId: generation.promptId ?? undefined,
promptName: generation.promptName ?? undefined,
promptVersion: generation.promptVersion ?? undefined,
};
})
: [];
@@ -10,7 +10,7 @@ import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context
import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
import { sessionsViewCols } from "@/src/server/api/definitions/sessionsView";
import { api } from "@/src/utils/api";
import { formatInterval, utcDateOffsetByDays } from "@/src/utils/dates";
import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
import { usdFormatter } from "@/src/utils/numbers";
import { type RouterOutput } from "@/src/utils/types";
import { useEffect } from "react";
@@ -156,7 +156,7 @@ export default function SessionsTable({
cell: ({ row }) => {
const value = row.getValue("sessionDuration");
return value && typeof value === "number"
? formatInterval(value)
? formatIntervalSeconds(value)
: undefined;
},
enableSorting: true,
@@ -17,7 +17,7 @@ import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context
import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
import { tracesTableColsWithOptions } from "@/src/server/api/definitions/tracesTable";
import { api } from "@/src/utils/api";
import { formatInterval, utcDateOffsetByDays } from "@/src/utils/dates";
import { formatIntervalSeconds, utcDateOffsetByDays } from "@/src/utils/dates";
import { type RouterInput, type RouterOutput } from "@/src/utils/types";
import { type Score } from "@prisma/client";
import { type RowSelectionState } from "@tanstack/react-table";
@@ -285,7 +285,7 @@ export default function TracesTable({
// add seconds to the end of the latency
cell: ({ row }) => {
const value: number | undefined = row.getValue("latency");
return value !== undefined ? formatInterval(value) : undefined;
return value !== undefined ? formatIntervalSeconds(value) : undefined;
},
enableHiding: true,
enableSorting: true,
@@ -21,7 +21,7 @@ import { NewDatasetItemFromObservationButton } from "@/src/features/datasets/com
import { type ObservationReturnType } from "@/src/server/api/routers/traces";
import { api } from "@/src/utils/api";
import { IOPreview } from "@/src/components/trace/IOPreview";
import { formatInterval } from "@/src/utils/dates";
import { formatIntervalSeconds } from "@/src/utils/dates";
import Link from "next/link";
import { usdFormatter } from "@/src/utils/numbers";
import { calculateDisplayTotalCost } from "@/src/components/trace";
@@ -70,7 +70,7 @@ export const ObservationPreview = (props: {
{preloadedObservation.completionStartTime ? (
<Badge variant="outline">
Time to first token:{" "}
{formatInterval(
{formatIntervalSeconds(
(preloadedObservation.completionStartTime.getTime() -
preloadedObservation.startTime.getTime()) /
1000,
@@ -80,7 +80,7 @@ export const ObservationPreview = (props: {
{preloadedObservation.endTime ? (
<Badge variant="outline">
Latency:{" "}
{formatInterval(
{formatIntervalSeconds(
(preloadedObservation.endTime.getTime() -
preloadedObservation.startTime.getTime()) /
1000,
+3 -3
View File
@@ -5,7 +5,7 @@ import { GroupedScoreBadges } from "@/src/components/grouped-score-badge";
import { Fragment } from "react";
import { type ObservationReturnType } from "@/src/server/api/routers/traces";
import { LevelColors } from "@/src/components/level-colors";
import { formatInterval } from "@/src/utils/dates";
import { formatIntervalSeconds } from "@/src/utils/dates";
import { MinusCircle, MinusIcon, PlusCircleIcon, PlusIcon } from "lucide-react";
import { Toggle } from "@/src/components/ui/toggle";
import { Button } from "@/src/components/ui/button";
@@ -96,7 +96,7 @@ const ObservationTreeTraceNode = (props: {
{props.showMetrics && props.trace.latency ? (
<div className="flex gap-2">
<span className="text-xs text-gray-500">
{formatInterval(props.trace.latency)}
{formatIntervalSeconds(props.trace.latency)}
</span>
</div>
) : null}
@@ -179,7 +179,7 @@ const ObservationTreeNode = (props: {
<div className="flex gap-2">
{observation.endTime ? (
<span className="text-xs text-gray-500">
{formatInterval(
{formatIntervalSeconds(
(observation.endTime.getTime() -
observation.startTime.getTime()) /
1000,
+4 -2
View File
@@ -21,7 +21,7 @@ import { ManualScoreButton } from "@/src/features/manual-scoring/components/Manu
import { Badge } from "@/src/components/ui/badge";
import { type ObservationReturnType } from "@/src/server/api/routers/traces";
import { IOPreview } from "@/src/components/trace/IOPreview";
import { formatInterval } from "@/src/utils/dates";
import { formatIntervalSeconds } from "@/src/utils/dates";
export const TracePreview = ({
trace,
@@ -45,7 +45,9 @@ export const TracePreview = ({
<CardDescription>{trace.timestamp.toLocaleString()}</CardDescription>
<div className="flex flex-wrap gap-2">
{!!trace.latency && (
<Badge variant="outline">{formatInterval(trace.latency)}</Badge>
<Badge variant="outline">
{formatIntervalSeconds(trace.latency)}
</Badge>
)}
<TraceAggUsageBadge observations={observations} />
{!!trace.release && (
+1 -1
View File
@@ -1 +1 @@
export const VERSION = "v2.7.2";
export const VERSION = "v2.9.0";
@@ -25,7 +25,15 @@ export const MetricTable = ({
{ column: "totalTokens", agg: "SUM" },
{ column: "model" },
],
filter: globalFilterState,
filter: [
...globalFilterState,
{
type: "string",
column: "type",
operator: "=",
value: "GENERATION",
},
],
groupBy: [{ type: "string", column: "model" }],
orderBy: [
{ column: "calculatedTotalCost", direction: "DESC", agg: "SUM" },
@@ -39,7 +39,10 @@ export const ModelUsageChart = ({
{ column: "calculatedTotalCost", agg: "SUM" },
{ column: "model" },
],
filter: globalFilterState,
filter: [
...globalFilterState,
{ type: "string", column: "type", operator: "=", value: "GENERATION" },
],
groupBy: [
{
type: "datetime",
@@ -37,7 +37,15 @@ export const UserChart = ({
{ column: "user" },
{ column: "traceId", agg: "COUNT" },
],
filter: globalFilterState,
filter: [
...globalFilterState,
{
type: "string",
column: "type",
operator: "=",
value: "GENERATION",
},
],
groupBy: [
{
type: "string",
@@ -3,7 +3,7 @@ import { DataTable } from "@/src/components/table/data-table";
import TableLink from "@/src/components/table/table-link";
import { type LangfuseColumnDef } from "@/src/components/table/types";
import { api } from "@/src/utils/api";
import { formatInterval, intervalInSeconds } from "@/src/utils/dates";
import { formatIntervalSeconds, intervalInSeconds } from "@/src/utils/dates";
import { type RouterOutput } from "@/src/utils/types";
import { type Score } from "@prisma/client";
@@ -69,7 +69,7 @@ export function DatasetRunItemsTable(
header: "Latency",
cell: ({ row }) => {
const latency: RowData["latency"] = row.getValue("latency");
return <>{formatInterval(latency)}</>;
return <>{formatIntervalSeconds(latency)}</>;
},
},
{
@@ -4,7 +4,7 @@ import TableLink from "@/src/components/table/table-link";
import { type LangfuseColumnDef } from "@/src/components/table/types";
import { useDetailPageLists } from "@/src/features/navigate-detail-pages/context";
import { api } from "@/src/utils/api";
import { formatInterval } from "@/src/utils/dates";
import { formatIntervalSeconds } from "@/src/utils/dates";
import { type RouterOutput } from "@/src/utils/types";
import { useEffect } from "react";
@@ -65,7 +65,7 @@ export function DatasetRunsTable(props: {
header: "Latency (avg)",
cell: ({ row }) => {
const avgLatency: RowData["avgLatency"] = row.getValue("avgLatency");
return <>{formatInterval(avgLatency)}</>;
return <>{formatIntervalSeconds(avgLatency)}</>;
},
},
{
@@ -3,15 +3,14 @@ import * as z from "zod";
import { zodResolver } from "@hookform/resolvers/zod";
import { useForm } from "react-hook-form";
import { useEffect, useState } from "react";
import JsonView from "react18-json-view";
import {
Form,
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/src/components/ui/form";
import { Textarea } from "@/src/components/ui/textarea";
import { Button } from "@/src/components/ui/button";
import { useHasAccess } from "@/src/features/rbac/utils/checkAccess";
@@ -108,7 +107,6 @@ export const EditDatasetItem = ({
// eslint-disable-next-line @typescript-eslint/no-misused-promises
onSubmit={form.handleSubmit(onSubmit)}
className="space-y-4"
onChange={() => setHasChanges(true)}
>
<div className="grid gap-4 md:grid-cols-2">
<FormField
@@ -117,13 +115,19 @@ export const EditDatasetItem = ({
render={({ field }) => (
<FormItem>
<FormLabel>Input</FormLabel>
<FormControl>
<Textarea
{...field}
className="min-h-[200px] font-mono text-xs"
disabled={!hasAccess}
/>
</FormControl>
<JsonView
src={JSON.parse(field.value || "{}") as unknown}
onEdit={(edit) => {
field.onChange(JSON.stringify(edit.src));
setHasChanges(true);
}}
onDelete={(edit) => {
field.onChange(JSON.stringify(edit.src));
setHasChanges(true);
}}
editable
className="rounded-md border border-gray-200 p-2 text-sm"
/>
<FormMessage />
</FormItem>
)}
@@ -134,13 +138,19 @@ export const EditDatasetItem = ({
render={({ field }) => (
<FormItem>
<FormLabel>Expected output (optional)</FormLabel>
<FormControl>
<Textarea
{...field}
className="min-h-[200px] font-mono text-xs"
disabled={!hasAccess}
/>
</FormControl>
<JsonView
src={JSON.parse(field.value || "{}") as unknown}
onEdit={(edit) => {
field.onChange(JSON.stringify(edit.src));
setHasChanges(true);
}}
onDelete={(edit) => {
field.onChange(JSON.stringify(edit.src));
setHasChanges(true);
}}
editable
className="rounded-md border border-gray-200 p-2 text-sm"
/>
<FormMessage />
</FormItem>
)}
@@ -2,6 +2,7 @@ import { Button } from "@/src/components/ui/button";
import * as z from "zod";
import { zodResolver } from "@hookform/resolvers/zod";
import { useForm } from "react-hook-form";
import JsonView from "react18-json-view";
import {
Form,
FormControl,
@@ -20,8 +21,8 @@ import {
import { api } from "@/src/utils/api";
import { useState } from "react";
import { usePostHog } from "posthog-js/react";
import { Textarea } from "@/src/components/ui/textarea";
import { type Prisma } from "@prisma/client";
import { jsonSchema } from "@/src/utils/zod";
const formSchema = z.object({
datasetId: z.string().min(1, "Select a dataset"),
@@ -35,8 +36,7 @@ const formSchema = z.object({
}
},
{
message:
"Invalid input. Please provide a JSON object or a string value enclosed in double quotes.",
message: "Invalid input. Please provide a JSON object",
},
),
expectedOutput: z.string().refine(
@@ -66,15 +66,14 @@ export const NewDatasetItemForm = (props: {
}) => {
const [formError, setFormError] = useState<string | null>(null);
const posthog = usePostHog();
const { observationInput, observationOutput } = props;
const form = useForm<z.infer<typeof formSchema>>({
resolver: zodResolver(formSchema),
defaultValues: {
datasetId: props.datasetId ?? "",
input: props.observationInput
? JSON.stringify(props.observationInput, null, 2)
: "",
expectedOutput: props.observationOutput
? JSON.stringify(props.observationOutput, null, 2)
input: observationInput ? JSON.stringify(observationInput, null, 2) : "",
expectedOutput: observationOutput
? JSON.stringify(observationOutput, null, 2)
: "",
},
});
@@ -146,12 +145,17 @@ export const NewDatasetItemForm = (props: {
render={({ field }) => (
<FormItem className="flex flex-col gap-2">
<FormLabel>Input</FormLabel>
<FormControl>
<Textarea
{...field}
className="min-h-[150px] flex-1 font-mono text-xs"
/>
</FormControl>
<JsonView
src={jsonSchema.parse(JSON.parse(field.value || "{}"))}
onEdit={(edit) => {
field.onChange(JSON.stringify(edit.src));
}}
onDelete={(edit) => {
field.onChange(JSON.stringify(edit.src));
}}
editable
className="rounded-md border border-gray-200 p-2 text-sm"
/>
<FormMessage />
</FormItem>
)}
@@ -162,12 +166,17 @@ export const NewDatasetItemForm = (props: {
render={({ field }) => (
<FormItem className="flex flex-col gap-2">
<FormLabel>Expected output</FormLabel>
<FormControl>
<Textarea
{...field}
className="min-h-[150px] flex-1 font-mono text-xs"
/>
</FormControl>
<JsonView
src={jsonSchema.parse(JSON.parse(field.value || "{}"))}
onEdit={(edit) => {
field.onChange(JSON.stringify(edit.src));
}}
onDelete={(edit) => {
field.onChange(JSON.stringify(edit.src));
}}
editable
className="rounded-md border border-gray-200 p-2 text-sm"
/>
<FormMessage />
</FormItem>
)}
@@ -19,6 +19,6 @@ export default async function feedbackApiHandler(
}
} catch (error) {
console.error(error);
res.status(400).json({ status: "Error" });
res.status(500).json({ status: "Error" });
}
}
@@ -178,7 +178,7 @@ export const NewPromptForm = (props: {
form.reset();
// go to the following page after creating the prompt
void router.push(
`/project/${props.projectId}/prompts/${newPrompt.name}`,
`/project/${props.projectId}/prompts/${encodeURIComponent(newPrompt.name)}`,
);
})
.catch((error) => {
@@ -56,7 +56,7 @@ export const PromptDetail = (props: PromptDetailProps) => {
},
{
name: prompt.name,
href: `/project/${props.projectId}/prompts/${prompt.name}`,
href: `/project/${props.projectId}/prompts/${props.promptName}`,
},
{ name: `Version ${prompt.version}` },
]}
@@ -89,7 +89,7 @@ export const PromptDetail = (props: PromptDetailProps) => {
/>
<DetailPageNav
key="nav"
currentId={prompt.name}
currentId={props.promptName}
path={(name) => `/project/${props.projectId}/prompts/${name}`}
listKey="prompts"
/>
@@ -35,7 +35,7 @@ export function PromptTable(props: { projectId: string }) {
if (prompts.isSuccess) {
setDetailPageList(
"prompts",
prompts.data.map((t) => t.name),
prompts.data.map((t) => encodeURIComponent(t.name)),
);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
@@ -49,7 +49,7 @@ export function PromptTable(props: { projectId: string }) {
const name: string = row.getValue("name");
return name && typeof name === "string" ? (
<TableLink
path={`/project/${props.projectId}/prompts/${name}`}
path={`/project/${props.projectId}/prompts/${encodeURIComponent(name)}`}
value={name}
truncateAt={30}
/>
@@ -6,6 +6,7 @@ import {
import { type ApiAccessScope } from "@/src/features/public-api/server/types";
import { prisma } from "@/src/server/db";
import { isPrismaException } from "@/src/utils/exceptions";
import * as Sentry from "@sentry/node";
export type AuthHeaderVerificationResult =
| {
@@ -66,6 +67,10 @@ export async function verifyAuthHeaderAndReturnScope(
throw new Error("Invalid credentials");
}
Sentry.setUser({
id: projectId,
});
return {
validKey: true,
scope: {
@@ -79,6 +84,9 @@ export async function verifyAuthHeaderAndReturnScope(
const publicKey = authHeader.replace("Bearer ", "");
const dbKey = await findDbKeyOrThrow(publicKey);
Sentry.setUser({
id: dbKey.projectId,
});
return {
validKey: true,
+1
View File
@@ -252,6 +252,7 @@ async function posthogTelemetry({
event: "telemetry",
properties: {
langfuseVersion: VERSION,
userDomains: domains,
totalProjects: totalProjects,
traces: countTraces,
scores: countScores,
+7 -1
View File
@@ -86,6 +86,12 @@ export default async function handler(
}
} catch (error: unknown) {
console.error(error);
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
if (isPrismaException(error)) {
return res.status(500).json({
error: "Internal Server Error",
@@ -93,7 +99,7 @@ export default async function handler(
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -71,9 +71,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -102,6 +102,12 @@ export default async function handler(
return res.status(200).json(runItem);
} catch (error: unknown) {
console.error(error);
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
if (isPrismaException(error)) {
return res.status(500).json({
error: "Internal Server Error",
@@ -109,7 +115,7 @@ export default async function handler(
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+7 -1
View File
@@ -61,6 +61,12 @@ export default async function handler(
}
} catch (error: unknown) {
console.error(error);
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
if (isPrismaException(error)) {
return res.status(500).json({
error: "Internal Server Error",
@@ -68,7 +74,7 @@ export default async function handler(
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -3,6 +3,7 @@ import { type NextApiRequest, type NextApiResponse } from "next";
import { z } from "zod";
import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
import { isPrismaException } from "@/src/utils/exceptions";
const DatasetsGetSchema = z.object({
name: z.string(),
@@ -78,9 +79,20 @@ export default async function handler(
return res.status(200).json(output);
} catch (error: unknown) {
console.error(error);
if (isPrismaException(error)) {
return res.status(500).json({
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -83,9 +83,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+8 -2
View File
@@ -11,7 +11,7 @@ import {
handleBatch,
handleBatchResultLegacy,
} from "@/src/pages/api/public/ingestion";
import { type z } from "zod";
import { z } from "zod";
import { isPrismaException } from "@/src/utils/exceptions";
export default async function handler(
@@ -67,6 +67,12 @@ export default async function handler(
handleBatchResultLegacy(result.errors, result.results, res);
} catch (error: unknown) {
console.error(error);
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
if (isPrismaException(error)) {
return res.status(500).json({
error: "Internal Server Error",
@@ -74,7 +80,7 @@ export default async function handler(
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+15 -3
View File
@@ -13,7 +13,7 @@ import {
handleBatch,
handleBatchResultLegacy,
} from "@/src/pages/api/public/ingestion";
import { type z } from "zod";
import { z } from "zod";
import { isPrismaException } from "@/src/utils/exceptions";
export default async function handler(
@@ -70,6 +70,12 @@ export default async function handler(
handleBatchResultLegacy(result.errors, result.results, res);
} catch (error: unknown) {
console.error(error);
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
if (isPrismaException(error)) {
return res.status(500).json({
error: "Internal Server Error",
@@ -77,7 +83,7 @@ export default async function handler(
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -129,6 +135,12 @@ export default async function handler(
handleBatchResultLegacy(result.errors, result.results, res);
} catch (error: unknown) {
console.error(error);
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
if (isPrismaException(error)) {
return res.status(500).json({
error: "Internal Server Error",
@@ -143,7 +155,7 @@ export default async function handler(
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+9 -2
View File
@@ -107,16 +107,23 @@ export default async function handler(
handleBatchResult([...errors, ...result.errors], result.results, res);
} catch (error: unknown) {
console.error(error);
if (isPrismaException(error)) {
return res.status(500).json({
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
console.error(error);
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
errors: [errorMessage],
});
+7 -1
View File
@@ -156,9 +156,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -63,9 +63,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -80,9 +80,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+14 -2
View File
@@ -78,9 +78,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -131,9 +137,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+15 -3
View File
@@ -70,9 +70,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -134,13 +140,19 @@ export default async function handler(
} catch (error: unknown) {
console.error(error);
if (isPrismaException(error)) {
return res.status(400).json({
return res.status(500).json({
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+68 -3
View File
@@ -5,7 +5,7 @@ import { cors, runMiddleware } from "@/src/features/public-api/server/cors";
import { verifyAuthHeaderAndReturnScope } from "@/src/features/public-api/server/apiAuth";
import { isPrismaException } from "@/src/utils/exceptions";
const ScoreDeleteSchema = z.object({
const ScoreSchema = z.object({
scoreId: z.string(),
});
@@ -33,7 +33,7 @@ export default async function handler(
});
}
const { scoreId } = ScoreDeleteSchema.parse(req.query); // uses query and not body
const { scoreId } = ScoreSchema.parse(req.query); // uses query and not body
const score = await prisma.score.findUnique({
select: {
@@ -70,9 +70,74 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
}
} else if (req.method === "GET") {
try {
// CHECK AUTH
const authCheck = await verifyAuthHeaderAndReturnScope(
req.headers.authorization,
);
if (!authCheck.validKey)
return res.status(401).json({
message: authCheck.error,
});
// END CHECK AUTH
const { scoreId } = ScoreSchema.parse(req.query);
// CHECK ACCESS SCOPE
if (authCheck.scope.accessLevel !== "all") {
return res.status(401).json({
message:
"Access denied - need to use basic auth with secret key to GET scores",
});
}
// END CHECK ACCESS SCOPE
const score = await prisma.score.findUnique({
where: {
id: scoreId,
trace: {
projectId: authCheck.scope.projectId,
},
},
});
if (!score) {
return res.status(404).json({
message: "Score not found within authorized project",
});
}
return res.status(200).json(score);
} catch (error: unknown) {
console.error(error);
if (isPrismaException(error)) {
return res.status(500).json({
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -69,9 +69,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+15 -4
View File
@@ -13,7 +13,7 @@ import {
handleBatch,
handleBatchResultLegacy,
} from "@/src/pages/api/public/ingestion";
import { type z } from "zod";
import { z } from "zod";
import { isPrismaException } from "@/src/utils/exceptions";
export default async function handler(
@@ -70,10 +70,16 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
console.error(error, req.body);
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -120,7 +126,12 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
if (error instanceof ResourceNotFoundError) {
return res.status(404).json({
message: "Span not found",
@@ -128,7 +139,7 @@ export default async function handler(
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+13 -3
View File
@@ -118,17 +118,21 @@ export default async function handler(
CONCAT('/project/', t.project_id,'/traces/',t.id) as "htmlPath",
t.timestamp,
t.name,
t.input,
t.output,
t.project_id as "projectId",
t.session_id as "sessionId",
t.metadata,
t.external_id as "externalId",
t.user_id as "userId",
t.release,
t.version,
t.public,
t.tags,
COALESCE(SUM(o.calculated_total_cost), 0)::DOUBLE PRECISION AS "totalCost",
COALESCE(EXTRACT(EPOCH FROM COALESCE(MAX(o."end_time"), MAX(o."start_time"))) - EXTRACT(EPOCH FROM MIN(o."start_time")), 0)::double precision AS "latency",
array_remove(array_agg(o.id), NULL) AS "observations",
array_remove(array_agg(s.id), NULL) AS "scores"
ARRAY_AGG(DISTINCT o.id) FILTER (WHERE o.id IS NOT NULL) AS "observations",
ARRAY_AGG(DISTINCT s.id) FILTER (WHERE s.id IS NOT NULL) AS "scores"
FROM "traces" AS t
LEFT JOIN "observations_view" AS o ON t.id = o.trace_id AND o.project_id = ${authCheck.scope.projectId}
LEFT JOIN "scores" AS s ON t.id = s.trace_id
@@ -168,9 +172,15 @@ export default async function handler(
errors: ["Internal Server Error"],
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+7 -1
View File
@@ -85,9 +85,15 @@ export default async function handler(
error: "Internal Server Error",
});
}
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
+8 -2
View File
@@ -119,15 +119,21 @@ export default async function handler(
return res.status(405).json({ message: "Method not allowed" });
}
} catch (error: unknown) {
console.error(error);
if (isPrismaException(error)) {
return res.status(500).json({
errors: ["Internal Server Error"],
});
}
console.error(error);
if (error instanceof z.ZodError) {
return res.status(400).json({
message: "Invalid request data",
error: error.errors,
});
}
const errorMessage =
error instanceof Error ? error.message : "An unknown error occurred";
res.status(400).json({
res.status(500).json({
message: "Invalid request data",
error: errorMessage,
});
@@ -43,7 +43,7 @@ export default function Dataset() {
datasetId={datasetId}
itemId={itemId}
/>
<Header title="Runs" />
<Header title="Runs" level="h3" />
<DatasetRunItemsTable
projectId={projectId}
datasetItemId={itemId}
@@ -31,4 +31,10 @@ export const sessionsViewCols: ColumnDefinition[] = [
type: "number",
internal: 't."countTraces"',
},
{
name: "Cost (USD)",
id: "totalCost",
type: "number",
internal: 'o."totalCost"',
},
];
@@ -114,10 +114,14 @@ export function getAllGenerationsSqlQuery({
o.calculated_input_cost as "calculatedInputCost",
o.calculated_output_cost as "calculatedOutputCost",
o.calculated_total_cost as "calculatedTotalCost",
o."latency"
o."latency",
o.prompt_id as "promptId",
p.name as "promptName",
p.version as "promptVersion"
FROM observations_view o
JOIN traces t ON t.id = o.trace_id AND t.project_id = o.project_id
LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = t.id and s_avg.observation_id = o.id
LEFT JOIN prompts p ON p.id = o.prompt_id
WHERE
o.project_id = ${input.projectId}
AND t.project_id = ${input.projectId}
@@ -29,17 +29,19 @@ export function transformStreamToJsonLines(): Transform {
const parsedOutput = outputSchema.safeParse(row.output);
if (parsedInput.success && parsedOutput.success) {
const output = JSON.stringify([
...parsedInput.data,
{
role: "assistant",
content:
typeof parsedOutput.data === "object" &&
"completion" in parsedOutput.data
? JSON.stringify(parsedOutput.data.completion)
: JSON.stringify(parsedOutput.data),
},
]);
const output = JSON.stringify({
messages: [
...parsedInput.data,
{
role: "assistant",
content:
typeof parsedOutput.data === "object" &&
"completion" in parsedOutput.data
? JSON.stringify(parsedOutput.data.completion)
: JSON.stringify(parsedOutput.data),
},
],
});
this.push(output + "\n");
}
@@ -22,7 +22,8 @@ export const getAllQuery = protectedProjectProcedure
(ObservationView & {
traceId: string;
traceName: string;
latency: number | null;
promptName: string | null;
promptVersion: string | null;
})[]
>(rawSqlQuery);
@@ -30,33 +31,27 @@ export const getAllQuery = protectedProjectProcedure
Array<{ count: bigint }>
>(
Prisma.sql`
WITH scores_avg AS (
SELECT
trace_id,
observation_id,
jsonb_object_agg(name::text, avg_value::double precision) AS scores_avg
FROM (
SELECT
trace_id,
observation_id,
name,
avg(value) avg_value
FROM
scores
GROUP BY
1,
2,
3
ORDER BY
1) tmp
GROUP BY
1, 2
)
SELECT
count(*)
FROM observations_view o
JOIN traces t ON t.id = o.trace_id AND t.project_id = o.project_id
LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = t.id and s_avg.observation_id = o.id
LEFT JOIN LATERAL (
SELECT
jsonb_object_agg(name::text, avg_value::double precision) AS "scores_avg"
FROM (
SELECT
name,
avg(value) avg_value
FROM
scores
WHERE
scores."trace_id" = t.id
AND scores."observation_id" = o.id
GROUP BY
name
) tmp
) AS s_avg ON true
WHERE
t.project_id = ${input.projectId}
AND o.type = 'GENERATION'
+40 -54
View File
@@ -463,64 +463,50 @@ function createTracesQuery(
orderByCondition: Sql,
) {
return Prisma.sql`
WITH usage AS (
SELECT
trace_id,
sum(prompt_tokens) AS "promptTokens",
sum(completion_tokens) AS "completionTokens",
sum(total_tokens) AS "totalTokens",
sum(calculated_total_cost) AS "calculatedTotalCost"
FROM
"observations_view"
WHERE
"trace_id" IS NOT NULL
AND "type" = 'GENERATION'
AND "project_id" = ${projectId}
${observationTimeseriesFilter}
GROUP BY
trace_id
),
trace_latency AS (
SELECT
trace_id,
EXTRACT(EPOCH FROM COALESCE(MAX("end_time"), MAX("start_time"))) - EXTRACT(EPOCH FROM MIN("start_time"))::double precision AS "latency"
FROM
"observations"
WHERE
"trace_id" IS NOT NULL
AND "project_id" = ${projectId}
${observationTimeseriesFilter}
GROUP BY
trace_id
),
-- used for filtering
scores_avg AS (
SELECT
trace_id,
jsonb_object_agg(name::text, avg_value::double precision) AS scores_avg
FROM (
SELECT
trace_id,
name,
avg(value) avg_value
FROM
scores
GROUP BY
1,
2
ORDER BY
1) tmp
GROUP BY
1
)
SELECT
${select}
FROM
"traces" AS t
LEFT JOIN usage AS u ON u.trace_id = t.id
-- used for filtering
LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = t.id
LEFT JOIN trace_latency AS tl ON tl.trace_id = t.id
LEFT JOIN LATERAL (
SELECT
SUM(prompt_tokens) AS "promptTokens",
SUM(completion_tokens) AS "completionTokens",
SUM(total_tokens) AS "totalTokens",
SUM(calculated_total_cost) AS "calculatedTotalCost"
FROM
"observations_view"
WHERE
trace_id = t.id
AND "type" = 'GENERATION'
AND "project_id" = ${projectId}
${observationTimeseriesFilter}
) AS u ON true
LEFT JOIN LATERAL (
SELECT
EXTRACT(EPOCH FROM COALESCE(MAX("end_time"), MAX("start_time"))) - EXTRACT(EPOCH FROM MIN("start_time"))::double precision AS "latency"
FROM
"observations"
WHERE
trace_id = t.id
AND "project_id" = ${projectId}
${observationTimeseriesFilter}
) AS tl ON true
LEFT JOIN LATERAL (
SELECT
jsonb_object_agg(name::text, avg_value::double precision) AS "scores_avg"
FROM (
SELECT
name,
AVG(value) avg_value
FROM
scores
WHERE
trace_id = t.id
GROUP BY
name
) tmp
) AS s_avg ON true
WHERE
t."project_id" = ${projectId}
${searchCondition}
@@ -31,6 +31,7 @@ export const executeQuery = async (
) => {
const query = sqlInterface.parse(unsafeQuery);
const sql = enrichAndCreateQuery(projectId, query);
const response = await prisma.$queryRaw<InternalDatabaseRow[]>(sql);
const parsedResult = outputParser(response);
+7 -1
View File
@@ -19,7 +19,7 @@ import { type Session } from "next-auth";
import { getServerAuthSession } from "@/src/server/auth";
import { DB, prisma } from "@/src/server/db";
import * as Sentry from "@sentry/node";
import * as z from "zod";
type CreateContextOptions = {
@@ -56,6 +56,12 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
// Get the session from the server using the getServerSession wrapper function
const session = await getServerAuthSession({ req, res });
Sentry.setUser({
id: session?.user?.id,
email: session?.user?.email ?? undefined,
username: session?.user?.name ?? undefined,
});
return createInnerTRPCContext({
session,
});
+1 -1
View File
@@ -18,7 +18,7 @@ export const setEndOfDay = (date: Date) => {
export const intervalInSeconds = (start: Date, end: Date | null) =>
end ? (end.getTime() - start.getTime()) / 1000 : 0;
export const formatInterval = (seconds: number) => {
export const formatIntervalSeconds = (seconds: number) => {
const hrs = Math.floor(seconds / 3600);
const mins = Math.floor((seconds % 3600) / 60);
const secs = Math.floor(seconds % 60);
+21 -31
View File
@@ -1,33 +1,23 @@
module.exports = {
"env": {
"browser": true,
"es2021": true
env: {
browser: true,
es2021: true,
},
extends: ["prettier"],
overrides: [
{
env: {
node: true,
},
files: [".eslintrc.{js,cjs}"],
parserOptions: {
sourceType: "script",
},
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended"
],
"overrides": [
{
"env": {
"node": true
},
"files": [
".eslintrc.{js,cjs}"
],
"parserOptions": {
"sourceType": "script"
}
}
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": "latest",
"sourceType": "module"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
}
}
],
parserOptions: {
ecmaVersion: "latest",
sourceType: "module",
},
rules: {},
};
+6
View File
@@ -0,0 +1,6 @@
# Ignore artifacts:
build
coverage
node_modules
dist
+1
View File
@@ -0,0 +1 @@
{}
+339
View File
@@ -0,0 +1,339 @@
import type { ColumnType } from "kysely";
export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
? ColumnType<S, I | undefined, U>
: ColumnType<T, T | undefined, T>;
export type Timestamp = ColumnType<Date, Date | string, Date | string>;
export const MembershipRole = {
OWNER: "OWNER",
ADMIN: "ADMIN",
MEMBER: "MEMBER",
VIEWER: "VIEWER"
} as const;
export type MembershipRole = (typeof MembershipRole)[keyof typeof MembershipRole];
export const ObservationType = {
SPAN: "SPAN",
EVENT: "EVENT",
GENERATION: "GENERATION"
} as const;
export type ObservationType = (typeof ObservationType)[keyof typeof ObservationType];
export const ObservationLevel = {
DEBUG: "DEBUG",
DEFAULT: "DEFAULT",
WARNING: "WARNING",
ERROR: "ERROR"
} as const;
export type ObservationLevel = (typeof ObservationLevel)[keyof typeof ObservationLevel];
export const PricingUnit = {
PER_1000_TOKENS: "PER_1000_TOKENS",
PER_1000_CHARS: "PER_1000_CHARS"
} as const;
export type PricingUnit = (typeof PricingUnit)[keyof typeof PricingUnit];
export const TokenType = {
PROMPT: "PROMPT",
COMPLETION: "COMPLETION",
TOTAL: "TOTAL"
} as const;
export type TokenType = (typeof TokenType)[keyof typeof TokenType];
export const DatasetStatus = {
ACTIVE: "ACTIVE",
ARCHIVED: "ARCHIVED"
} as const;
export type DatasetStatus = (typeof DatasetStatus)[keyof typeof DatasetStatus];
export type Account = {
id: string;
user_id: string;
type: string;
provider: string;
providerAccountId: string;
refresh_token: string | null;
access_token: string | null;
expires_at: number | null;
expires_in: number | null;
ext_expires_in: number | null;
token_type: string | null;
scope: string | null;
id_token: string | null;
session_state: string | null;
};
export type ApiKey = {
id: string;
created_at: Generated<Timestamp>;
note: string | null;
public_key: string;
hashed_secret_key: string;
fast_hashed_secret_key: string | null;
display_secret_key: string;
last_used_at: Timestamp | null;
expires_at: Timestamp | null;
project_id: string;
};
export type AuditLog = {
id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
user_id: string;
project_id: string;
user_project_role: MembershipRole;
resource_type: string;
resource_id: string;
action: string;
before: string | null;
after: string | null;
};
export type CronJobs = {
name: string;
last_run: Timestamp | null;
job_started_at: Timestamp | null;
state: string | null;
};
export type Dataset = {
id: string;
name: string;
project_id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
};
export type DatasetItem = {
id: string;
status: Generated<DatasetStatus>;
input: unknown;
expected_output: unknown | null;
source_observation_id: string | null;
dataset_id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
};
export type DatasetRunItems = {
id: string;
dataset_run_id: string;
dataset_item_id: string;
observation_id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
};
export type DatasetRuns = {
id: string;
name: string;
dataset_id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
};
export type Events = {
id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
project_id: string;
data: unknown;
headers: Generated<unknown>;
url: string | null;
method: string | null;
};
export type Example = {
id: string;
created_at: Generated<Timestamp>;
updated_at: Timestamp;
};
export type Membership = {
project_id: string;
user_id: string;
role: MembershipRole;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
};
export type MembershipInvitation = {
id: string;
email: string;
role: MembershipRole;
project_id: string;
sender_id: string | null;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
};
export type Model = {
id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
project_id: string | null;
model_name: string;
match_pattern: string;
start_date: Timestamp | null;
input_price: string | null;
output_price: string | null;
total_price: string | null;
unit: string;
tokenizer_id: string | null;
tokenizer_config: unknown | null;
};
export type Observation = {
id: string;
trace_id: string | null;
project_id: string;
type: ObservationType;
start_time: Generated<Timestamp>;
end_time: Timestamp | null;
name: string | null;
metadata: unknown | null;
parent_observation_id: string | null;
level: Generated<ObservationLevel>;
status_message: string | null;
version: string | null;
created_at: Generated<Timestamp>;
model: string | null;
internal_model: string | null;
modelParameters: unknown | null;
input: unknown | null;
output: unknown | null;
prompt_tokens: Generated<number>;
completion_tokens: Generated<number>;
total_tokens: Generated<number>;
unit: string | null;
input_cost: string | null;
output_cost: string | null;
total_cost: string | null;
completion_start_time: Timestamp | null;
prompt_id: string | null;
};
export type ObservationView = {
id: string;
trace_id: string | null;
project_id: string;
type: ObservationType;
start_time: Generated<Timestamp>;
end_time: Timestamp | null;
name: string | null;
metadata: unknown | null;
parent_observation_id: string | null;
level: Generated<ObservationLevel>;
status_message: string | null;
version: string | null;
created_at: Generated<Timestamp>;
model: string | null;
modelParameters: unknown | null;
input: unknown | null;
output: unknown | null;
prompt_tokens: Generated<number>;
completion_tokens: Generated<number>;
total_tokens: Generated<number>;
unit: string | null;
completion_start_time: Timestamp | null;
prompt_id: string | null;
model_id: string | null;
input_price: string | null;
output_price: string | null;
total_price: string | null;
calculated_input_cost: string | null;
calculated_output_cost: string | null;
calculated_total_cost: string | null;
latency: number | null;
};
export type Pricing = {
id: string;
model_name: string;
pricing_unit: Generated<PricingUnit>;
price: string;
currency: Generated<string>;
token_type: TokenType;
};
export type Project = {
id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
name: string;
cloud_config: unknown | null;
};
export type Prompt = {
id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
project_id: string;
created_by: string;
prompt: string;
name: string;
version: number;
is_active: boolean;
config: Generated<unknown>;
};
export type Score = {
id: string;
timestamp: Generated<Timestamp>;
name: string;
value: number;
comment: string | null;
trace_id: string;
observation_id: string | null;
};
export type Session = {
id: string;
session_token: string;
user_id: string;
expires: Timestamp;
};
export type Trace = {
id: string;
external_id: string | null;
timestamp: Generated<Timestamp>;
name: string | null;
user_id: string | null;
metadata: unknown | null;
release: string | null;
version: string | null;
project_id: string;
public: Generated<boolean>;
bookmarked: Generated<boolean>;
tags: Generated<string[]>;
input: unknown | null;
output: unknown | null;
session_id: string | null;
};
export type TraceSession = {
id: string;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
project_id: string;
bookmarked: Generated<boolean>;
public: Generated<boolean>;
};
export type User = {
id: string;
name: string | null;
email: string | null;
email_verified: Timestamp | null;
password: string | null;
image: string | null;
admin: Generated<boolean>;
created_at: Generated<Timestamp>;
updated_at: Generated<Timestamp>;
feature_flags: Generated<string[]>;
};
export type VerificationToken = {
identifier: string;
token: string;
expires: Timestamp;
};
export type DB = {
Account: Account;
api_keys: ApiKey;
audit_logs: AuditLog;
cron_jobs: CronJobs;
dataset_items: DatasetItem;
dataset_run_items: DatasetRunItems;
dataset_runs: DatasetRuns;
datasets: Dataset;
events: Events;
Example: Example;
membership_invitations: MembershipInvitation;
memberships: Membership;
models: Model;
observations: Observation;
observations_view: ObservationView;
pricings: Pricing;
projects: Project;
prompts: Prompt;
scores: Score;
Session: Session;
trace_sessions: TraceSession;
traces: Trace;
users: User;
verification_tokens: VerificationToken;
};
+2768 -50
View File
File diff suppressed because it is too large Load Diff
+13 -5
View File
@@ -5,10 +5,13 @@
"main": "index.js",
"private": true,
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"test": "vitest run",
"coverage": "vitest run --coverage",
"build": "tsc -p tsconfig.json",
"start": "node dist/index.js",
"dev": "nodemon src/index.ts"
"dev": "nodemon src/index.ts",
"lint": "eslint .",
"lint:fix": "eslint --fix ."
},
"author": "",
"license": "MIT",
@@ -24,13 +27,18 @@
"devDependencies": {
"@types/node": "^20.11.19",
"@types/pg": "^8.11.0",
"@typescript-eslint/eslint-plugin": "^7.0.2",
"@typescript-eslint/parser": "^7.0.2",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"eslint": "^8.56.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-n": "^16.6.2",
"eslint-plugin-promise": "^6.1.1",
"kysely-codegen": "^0.11.0",
"nodemon": "^3.0.3",
"pino-pretty": "^10.3.1",
"prettier": "3.2.5",
"ts-node": "^10.9.2",
"typescript": "^5.3.3"
"typescript": "^5.3.3",
"vitest": "^1.3.1"
}
}
+6
View File
@@ -0,0 +1,6 @@
import { expect, test } from "vitest";
import { sum } from "../sum";
test("adds 1 + 2 to equal 3", () => {
expect(sum(1, 2)).toBe(3);
});
+10 -10
View File
@@ -1,11 +1,11 @@
// import { Kysely, PostgresDialect } from "kysely";
// import { DB } from "../../web/prisma/generated/types";
// import { Pool } from "pg";
import { Kysely, PostgresDialect } from "kysely";
import { Pool } from "pg";
import { DB } from "../generated/types";
// export const db = new Kysely<DB>({
// dialect: new PostgresDialect({
// pool: new Pool({
// connectionString: "postgres://postgres:password@localhost:5432/postgres",
// }),
// }),
// });
export const db = new Kysely<DB>({
dialect: new PostgresDialect({
pool: new Pool({
connectionString: process.env.DATABASE_URL,
}),
}),
});
+3 -9
View File
@@ -3,15 +3,12 @@ import consumer from "./redis-consumer";
import { getLogger } from "./logger";
import redis from "@fastify/redis";
import { db } from "./database";
const fastify = Fastify({
logger: getLogger("development") ?? true, // defaults to true if no entry matches in the map
});
console.log(
"trying to connect to redis",
process.env.REDIS_URL,
process.env.REDIS_PORT
);
fastify.register(redis, {
host: process.env.REDIS_URL,
port: process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : 6379,
@@ -31,12 +28,9 @@ const start = async () => {
process.exit(1);
}
};
start();
fastify.get("/", async (request, reply) => {
console.log("Hello world!");
// const { redis } = fastify;
// redis.set("mykey", "value");
// console.log(await redis.get("mykey"));
return { hello: "world" };
});
+1 -2
View File
@@ -2,13 +2,12 @@ import { FastifyInstance, FastifyPluginOptions } from "fastify";
import fp from "fastify-plugin";
async function routes(fastify: FastifyInstance, options: FastifyPluginOptions) {
console.log("routes");
fastify.redis.subscribe("ingestion", (err, count) => {
if (err) {
fastify.log.error("Redis error:", err);
} else {
fastify.log.info(
`Subscribed to ${count} channel. Listening for updates on the channel...`
`Subscribed to ${count} channel. Listening for updates on the channel...`,
);
}
});
+4
View File
@@ -0,0 +1,4 @@
// sum.js
export function sum(a: number, b: number) {
return a + b;
}
+9 -8
View File
@@ -1,11 +1,12 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
"target": "es2017" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
"module": "commonjs" /* Specify what module code is generated. */,
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
"skipLibCheck": true /* Skip type checking all .d.ts files. */,
"outDir": "dist"
}
"target": "es2017",
"module": "commonjs",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"skipLibCheck": true,
"outDir": "dist",
"strictNullChecks": true // Add this line
},
"include": ["src/**/*", ".eslintrc.js"]
}