chore: release v2.7.2

.github/workflows/pipeline.yml

@@ -41,26 +41,22 @@ jobs:
       run:
         working-directory: web
     runs-on: ubuntu-latest
-    env:
-      REGISTRY: ghcr.io
-      IMAGE_NAME: ${{ github.repository }}
-
     steps:
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          cache-dependency-path: "**/package-lock.json"
-
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Build Docker image
+      - name: Build Web Docker image
         uses: docker/build-push-action@v4
         with:
           context: web
           push: false
 
+      - name: Build Worker Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: worker
+          push: false
+
   tests:
     defaults:
       run:

.vscode/settings.json

@@ -21,13 +21,11 @@
     "**/*.{cjs,mjs,ts,tsx,astro,md,mdx,json,yaml,yml}"
   ],
   "mdx.experimentalLanguageServer": true,
-  "[astro]": {
-    "editor.defaultFormatter": "astro-build.astro-vscode"
-  },
   "typescript.preferences.importModuleSpecifier": "non-relative",
   "docwriter.style": "JSDoc",
   "[prisma]": {
     "editor.defaultFormatter": "Prisma.prisma"
   },
-  "eslint.lintTask.enable": true
+  "eslint.lintTask.enable": true,
+  "eslint.workingDirectories": ["./web", "./worker"]
 }

CONTRIBUTING.md

@@ -93,7 +93,7 @@ The diagram below may not show all relationships if the foreign key is not defin
 
 Full database schema: [prisma/schema.prisma](prisma/schema.prisma)
 
-<img src="./prisma/database.svg">
+<img src="./web/prisma/database.svg">
 
 ### Infrastructure & Network Overview
 
@@ -123,7 +123,7 @@ Requirements
 2. Run the development database
 
    ```bash
-   docker-compose -f docker-compose.dev.yml up -d
+   npm run infra:dev:up
    ```
 
 3. Create an env file

docker-compose.build.yml

@@ -1,11 +1,13 @@
 version: "3.5"
 
 services:
-  langfuse-server:
+  server:
     build:
-      dockerfile: Dockerfile
+      dockerfile: ./Dockerfile
+      context: ./web
     depends_on:
       - db
+      - redis
     ports:
       - "3000:3000"
     environment:
@@ -16,8 +18,40 @@ services:
       - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
       - NEXT_PUBLIC_SIGN_UP_DISABLED=${NEXT_PUBLIC_SIGN_UP_DISABLED:-false}
       - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      - REDIS_URL=${REDIS_URL:-redis}
+      - REDIS_PORT=${REDIS_PORT:-6379}
+      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
     restart: always
 
+  worker:
+    build:
+      dockerfile: ./Dockerfile
+      context: ./worker
+    depends_on:
+      - db
+      - redis
+      - server
+    ports:
+      - "3030:3030"
+    environment:
+      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
+      - NEXTAUTH_SECRET=mysecret
+      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
+      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      - PORT=${PORT:-3030}
+      - REDIS_URL=${REDIS_URL:-redis}
+      - REDIS_PORT=${REDIS_PORT:-6379}
+      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+    restart: always
+
+  redis:
+    image: redis
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    restart: always
+    ports:
+      - 6379:6379
+
   db:
     image: postgres
     restart: always

docker-compose.dev.yml

@@ -1,6 +1,12 @@
 version: "3.5"
 
 services:
+  redis:
+    image: redis
+    restart: always
+    ports:
+      - 6379:6379
+
   db:
     image: postgres
     restart: always

package-lock.json

@@ -7,6 +7,7 @@
     "": {
       "name": "langfuse",
       "version": "2.7.0",
+      "hasInstallScript": true,
       "license": "MIT",
       "devDependencies": {
         "@types/eslint": "^8.56.2",

package.json

@@ -2,8 +2,10 @@
   "name": "langfuse",
   "version": "2.7.0",
   "scripts": {
-    "infra:dev:up": "docker-compose  -f ../docker-compose.dev.yml up -d",
-    "infra:dev:down": "docker-compose -f ../docker-compose.dev.yml down"
+    "infra:dev:up": "docker-compose -f ../docker-compose.dev.yml up -d",
+    "infra:dev:down": "docker-compose -f ../docker-compose.dev.yml down",
+    "dev": "npm run dev --prefix worker/ & npm run dev --prefix web/",
+    "postinstall": "cd worker && npm install && cd ../web && npm install"
   },
   "author": "",
   "license": "MIT",

web/.dockerignore

@@ -0,0 +1,7 @@
+Dockerfile
+.dockerignore
+node_modules
+npm-debug.log
+README.md
+.next
+.git

web/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "langfuse-core",
-  "version": "2.7.1",
+  "version": "2.7.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "langfuse-core",
-      "version": "2.7.1",
+      "version": "2.7.2",
       "hasInstallScript": true,
       "dependencies": {
         "@anthropic-ai/tokenizer": "^0.0.4",
@@ -78,7 +78,7 @@
         "react-hook-form": "^7.50.1",
         "react-icons": "^5.0.1",
         "react-responsive": "^9.0.2",
-        "react18-json-view": "^0.2.7",
+        "react18-json-view": "^0.2.8-canary.1",
         "sonner": "^1.4.0",
         "superjson": "2.2.1",
         "tailwind-merge": "^2.2.1",
@@ -16946,9 +16946,9 @@
       }
     },
     "node_modules/react18-json-view": {
-      "version": "0.2.7",
-      "resolved": "https://registry.npmjs.org/react18-json-view/-/react18-json-view-0.2.7.tgz",
-      "integrity": "sha512-u1H0ZrjrYZUuXEyjcMU+/lVbFi890SNacTcYQT+e7/TI7OeczHyLwcngY4JbtZgfhwjIU078O1+NKh97IVVwZw==",
+      "version": "0.2.8-canary.1",
+      "resolved": "https://registry.npmjs.org/react18-json-view/-/react18-json-view-0.2.8-canary.1.tgz",
+      "integrity": "sha512-/kza8+jEl21rXfmmCq3SHdQLTrFqV5jtjK4O0ly6khtjFplvudXnsHQ64AWmaJ3x7wgx/6A7N27xgIznT+GAVA==",
       "peerDependencies": {
         "react": ">=16.8.0"
       }

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse-core",
-  "version": "2.7.1",
+  "version": "2.7.2",
   "private": true,
   "scripts": {
     "prebuild": "cp generated/openapi-client/openapi.yml public/openapi-client.yml && cp generated/openapi-server/openapi.yml public/openapi-server.yml",
@@ -96,7 +96,7 @@
     "react-hook-form": "^7.50.1",
     "react-icons": "^5.0.1",
     "react-responsive": "^9.0.2",
-    "react18-json-view": "^0.2.7",
+    "react18-json-view": "^0.2.8-canary.1",
     "sonner": "^1.4.0",
     "superjson": "2.2.1",
     "tailwind-merge": "^2.2.1",

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.7.1";
+export const VERSION = "v2.7.2";

web/tsconfig.json

@@ -34,4 +34,4 @@
     "types/global.d.ts"
   ],
   "exclude": ["node_modules", "sdk"]
-}
+}

worker/.dockerignore

@@ -0,0 +1,7 @@
+Dockerfile
+.dockerignore
+node_modules
+npm-debug.log
+README.md
+.next
+.git

worker/.eslintrc.js

@@ -0,0 +1,33 @@
+module.exports = {
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:@typescript-eslint/recommended"
+    ],
+    "overrides": [
+        {
+            "env": {
+                "node": true
+            },
+            "files": [
+                ".eslintrc.{js,cjs}"
+            ],
+            "parserOptions": {
+                "sourceType": "script"
+            }
+        }
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaVersion": "latest",
+        "sourceType": "module"
+    },
+    "plugins": [
+        "@typescript-eslint"
+    ],
+    "rules": {
+    }
+}

worker/.gitignore

@@ -0,0 +1,148 @@
+# Created by https://www.toptal.com/developers/gitignore/api/node
+# Edit at https://www.toptal.com/developers/gitignore?templates=node
+
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+### Node Patch ###
+# Serverless Webpack directories
+.webpack/
+
+# Optional stylelint cache
+
+# SvelteKit build / generate output
+.svelte-kit
+
+# End of https://www.toptal.com/developers/gitignore/api/node
+
+dist
+
+/dist

worker/Dockerfile

@@ -0,0 +1,35 @@
+FROM node:20-alpine AS base
+
+WORKDIR /app
+
+# Install dependencies based on the preferred package manager
+COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* ./
+RUN \
+  if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
+  elif [ -f package-lock.json ]; then npm ci; \
+  elif [ -f pnpm-lock.yaml ]; then yarn global add pnpm && pnpm i --frozen-lockfile; \
+  else echo "Lockfile not found." && exit 1; \
+  fi
+
+
+FROM base AS builder
+
+# Copying the entire context to /app including package.json
+COPY . /app
+
+ENV NODE_ENV production
+
+RUN npm run build
+
+FROM base AS runner
+
+ENV NODE_ENV production
+
+# Copy the built application from the builder stage
+COPY --from=builder /app/dist ./dist
+
+EXPOSE 3030
+
+ENV ADDRESS=0.0.0.0 PORT=3030
+
+CMD ["npm", "start"]

worker/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "worker",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "private": true,
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "tsc -p tsconfig.json",
+    "start": "node dist/index.js",
+    "dev": "nodemon src/index.ts"
+  },
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@fastify/redis": "^6.1.1",
+    "backoff": "^2.5.0",
+    "fastify": "^4.26.1",
+    "kysely": "^0.27.2",
+    "node": "^21.6.2",
+    "pg": "^8.11.3",
+    "zod": "^3.22.4"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.19",
+    "@types/pg": "^8.11.0",
+    "@typescript-eslint/eslint-plugin": "^7.0.2",
+    "@typescript-eslint/parser": "^7.0.2",
+    "eslint": "^8.56.0",
+    "kysely-codegen": "^0.11.0",
+    "nodemon": "^3.0.3",
+    "pino-pretty": "^10.3.1",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3"
+  }
+}

worker/src/database.ts

@@ -0,0 +1,11 @@
+// import { Kysely, PostgresDialect } from "kysely";
+// import { DB } from "../../web/prisma/generated/types";
+// import { Pool } from "pg";
+
+// export const db = new Kysely<DB>({
+//   dialect: new PostgresDialect({
+//     pool: new Pool({
+//       connectionString: "postgres://postgres:password@localhost:5432/postgres",
+//     }),
+//   }),
+// });

worker/src/index.ts

@@ -0,0 +1,42 @@
+import Fastify from "fastify";
+import consumer from "./redis-consumer";
+
+import { getLogger } from "./logger";
+import redis from "@fastify/redis";
+
+const fastify = Fastify({
+  logger: getLogger("development") ?? true, // defaults to true if no entry matches in the map
+});
+console.log(
+  "trying to connect to redis",
+  process.env.REDIS_URL,
+  process.env.REDIS_PORT
+);
+fastify.register(redis, {
+  host: process.env.REDIS_URL,
+  port: process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : 6379,
+  password: process.env.REDIS_AUTH,
+});
+fastify.register(consumer);
+
+const start = async () => {
+  try {
+    // listen to 0.0.0.0 is required for docker
+    await fastify.listen({
+      port: process.env.PORT ? parseInt(process.env.PORT) : 3030,
+      host: "0.0.0.0",
+    });
+  } catch (err) {
+    fastify.log.error(err);
+    process.exit(1);
+  }
+};
+start();
+
+fastify.get("/", async (request, reply) => {
+  console.log("Hello world!");
+  // const { redis } = fastify;
+  // redis.set("mykey", "value");
+  // console.log(await redis.get("mykey"));
+  return { hello: "world" };
+});

worker/src/logger.ts

@@ -0,0 +1,17 @@
+export const getLogger = (env: "development" | "production" | "test") => {
+  return envToLogger[env] ?? true;
+};
+
+const envToLogger = {
+  development: {
+    transport: {
+      target: "pino-pretty",
+      options: {
+        translateTime: "HH:MM:ss Z",
+        ignore: "pid,hostname",
+      },
+    },
+  },
+  production: true,
+  test: false,
+};

worker/src/redis-consumer.ts

@@ -0,0 +1,21 @@
+import { FastifyInstance, FastifyPluginOptions } from "fastify";
+import fp from "fastify-plugin";
+
+async function routes(fastify: FastifyInstance, options: FastifyPluginOptions) {
+  console.log("routes");
+  fastify.redis.subscribe("ingestion", (err, count) => {
+    if (err) {
+      fastify.log.error("Redis error:", err);
+    } else {
+      fastify.log.info(
+        `Subscribed to ${count} channel. Listening for updates on the channel...`
+      );
+    }
+  });
+
+  fastify.redis.on("message", (channel, message) => {
+    fastify.log.info(`Received "${message}" from ${channel}`);
+  });
+}
+
+export default fp(routes);

worker/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+    "target": "es2017" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    "module": "commonjs" /* Specify what module code is generated. */,
+    "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
+    "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
+    "skipLibCheck": true /* Skip type checking all .d.ts files. */,
+    "outDir": "dist"
+  }
+}