mirror of
https://github.com/hanzoai/insights.git
synced 2026-11-03 00:52:32 +00:00
zeekay
bc4ddf332b
Mechanically re-derived: bin/debrand runs over a fresh import of posthog/posthog@fa1a9a60d4 (2026-08-06). The pipeline (.hanzo/workflows), the Dockerfiles and bin/docker* entrypoints, compose files, LLM.md and the squashed initial migration are restored from the shipped tree, same as the previous import. The prune's import-closure guard kept 21 products the new core now imports; dropping them is a refactor now, recorded rather than forced.
234 lines
12 KiB
JSON
234 lines
12 KiB
JSON
{
|
||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||
"properties": {
|
||
"conditions": {
|
||
"description": "Trigger conditions that filter which events are evaluated. OR between condition sets, AND within each. Each set is {id, rollout_percentage, properties[]} — `rollout_percentage` (0-100, defaults to 100) is the sampling field the dispatcher reads.",
|
||
"items": {
|
||
"description": "A trigger condition set controlling which generations an evaluation runs on.",
|
||
"properties": {
|
||
"id": {
|
||
"description": "Stable identifier for this condition set.",
|
||
"maxLength": 100,
|
||
"type": "string"
|
||
},
|
||
"properties": {
|
||
"description": "Property filters (event or person) that scope which generations match this condition set.",
|
||
"items": {
|
||
"additionalProperties": {},
|
||
"propertyNames": {
|
||
"type": "string"
|
||
},
|
||
"type": "object"
|
||
},
|
||
"type": "array"
|
||
},
|
||
"rollout_percentage": {
|
||
"default": 100,
|
||
"description": "Percentage (0-100) of matching events to sample for this evaluation. Defaults to 100.",
|
||
"maximum": 100,
|
||
"minimum": 0,
|
||
"type": "number"
|
||
}
|
||
},
|
||
"required": ["id"],
|
||
"type": "object"
|
||
},
|
||
"type": "array"
|
||
},
|
||
"deleted": {
|
||
"description": "Set to true to soft-delete the evaluation.",
|
||
"type": "boolean"
|
||
},
|
||
"description": {
|
||
"description": "Optional description of what this evaluation checks.",
|
||
"type": "string"
|
||
},
|
||
"directory_id": {
|
||
"anyOf": [
|
||
{
|
||
"type": "string"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
],
|
||
"description": "Directory containing the evaluation. Pass null to move the evaluation to the top level."
|
||
},
|
||
"enabled": {
|
||
"description": "Whether the evaluation runs automatically on new $ai_generation events.",
|
||
"type": "boolean"
|
||
},
|
||
"evaluation_config": {
|
||
"anyOf": [
|
||
{
|
||
"properties": {
|
||
"prompt": {
|
||
"description": "Evaluation criteria for the LLM judge. Describe what makes a good vs bad response.",
|
||
"minLength": 1,
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": ["prompt"],
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"source": {
|
||
"description": "Script source code. Must return true (pass), false (fail), or null for N/A.",
|
||
"minLength": 1,
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": ["source"],
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"source": {
|
||
"default": "user_messages",
|
||
"description": "Classify sentiment from user messages in the generation input.",
|
||
"enum": ["user_messages"],
|
||
"type": "string"
|
||
}
|
||
},
|
||
"type": "object"
|
||
}
|
||
],
|
||
"description": "Configuration dict. For 'llm_judge': {prompt}; for 'script': {source}; for 'sentiment': {source: 'user_messages'}."
|
||
},
|
||
"evaluation_type": {
|
||
"description": "'llm_judge' uses an LLM to score outputs against a prompt; 'script' runs deterministic Script code; 'sentiment' classifies user-message sentiment.\n\n* `llm_judge` - LLM as a judge\n* `script` - Script\n* `sentiment` - Sentiment analysis",
|
||
"enum": ["llm_judge", "script", "sentiment"],
|
||
"type": "string"
|
||
},
|
||
"model_configuration": {
|
||
"anyOf": [
|
||
{
|
||
"description": "Nested serializer for model configuration.",
|
||
"properties": {
|
||
"model": {
|
||
"maxLength": 100,
|
||
"type": "string"
|
||
},
|
||
"provider": {
|
||
"description": "* `openai` - Openai\n* `anthropic` - Anthropic\n* `gemini` - Gemini\n* `openrouter` - Openrouter\n* `fireworks` - Fireworks\n* `azure_openai` - Azure OpenAI\n* `together_ai` - Together AI\n* `minimax` - MiniMax\n* `zeabur` - Zeabur AI Hub",
|
||
"enum": [
|
||
"openai",
|
||
"anthropic",
|
||
"gemini",
|
||
"openrouter",
|
||
"fireworks",
|
||
"azure_openai",
|
||
"together_ai",
|
||
"minimax",
|
||
"zeabur"
|
||
],
|
||
"type": "string"
|
||
},
|
||
"provider_key_id": {
|
||
"anyOf": [
|
||
{
|
||
"type": "string"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
],
|
||
"description": "Optional team provider key to run this evaluation with; it must use the same provider. May be null when no key is pinned or after the selected key is removed."
|
||
},
|
||
"provider_key_name": {
|
||
"anyOf": [
|
||
{
|
||
"type": "string"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
]
|
||
}
|
||
},
|
||
"required": ["provider", "model"],
|
||
"type": "object"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
],
|
||
"description": "Provider and model for an llm_judge evaluation. Required when creating or switching to llm_judge. To add or replace a model, provide both provider and model. On an existing configured llm_judge, omit this field to keep the current model; null is rejected. When switching an llm_judge to script or sentiment, set this field to null. Legacy llm_judge evaluations without a model remain editable without adding one. The nested provider_key_id may be null."
|
||
},
|
||
"name": {
|
||
"description": "Name of the evaluation.",
|
||
"maxLength": 400,
|
||
"type": "string"
|
||
},
|
||
"output_config": {
|
||
"description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results.",
|
||
"properties": {
|
||
"allows_na": {
|
||
"default": false,
|
||
"description": "Whether the evaluation can return N/A for non-applicable generations.",
|
||
"type": "boolean"
|
||
}
|
||
},
|
||
"type": "object"
|
||
},
|
||
"output_type": {
|
||
"description": "Output format. Use 'boolean' for pass/fail evaluations and 'sentiment' for sentiment analysis.\n\n* `boolean` - Boolean (Pass/Fail)\n* `sentiment` - Sentiment",
|
||
"enum": ["boolean", "sentiment"],
|
||
"type": "string"
|
||
},
|
||
"target": {
|
||
"description": "What the evaluation runs on. 'generation' evaluates each matching $ai_generation event individually. 'trace' evaluates the whole trace once and 'session' the whole $ai_session_id session once: the first matching generation schedules a run that waits for the unit to settle, then evaluates all of its events together. Condition filters still match individual generations — a unit is evaluated when any of its generations matches, and sampling applies per unit. A 'session' evaluation only fires for generations that carry $ai_session_id. When and how the run fires is controlled by target_config's settle strategy.\n\n* `generation` - Generation\n* `trace` - Trace\n* `session` - Session",
|
||
"enum": ["generation", "trace", "session"],
|
||
"type": "string"
|
||
},
|
||
"target_config": {
|
||
"anyOf": [
|
||
{
|
||
"properties": {
|
||
"strategy": {
|
||
"description": "Wait a fixed window after the first matching generation, then evaluate.",
|
||
"enum": ["fixed_window"],
|
||
"type": "string"
|
||
},
|
||
"window_seconds": {
|
||
"description": "Seconds to wait after the first matching generation before evaluating the whole unit. Captured when the run is scheduled — editing it does not change runs already in flight. The accepted range depends on `target`: 10–7200 for 'trace', 10–604800 for 'session'. The default also depends on `target`; see the field-level help_text.",
|
||
"maximum": 604800,
|
||
"minimum": 10,
|
||
"type": "number"
|
||
}
|
||
},
|
||
"required": ["strategy"],
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"max_age_seconds": {
|
||
"description": "Hard cap in seconds on the total wait from the first matching generation, even if the unit stays active. Must be at least quiet_period_seconds. The accepted range depends on `target`: 60–7200 for 'trace', 60–604800 for 'session'. The default also depends on `target`; see the field-level help_text.",
|
||
"maximum": 604800,
|
||
"minimum": 60,
|
||
"type": "number"
|
||
},
|
||
"quiet_period_seconds": {
|
||
"description": "Seconds without new activity before the unit counts as settled. The accepted range depends on `target`: 10–1800 for 'trace', 10–86400 for 'session'. The default also depends on `target`; see the field-level help_text.",
|
||
"maximum": 86400,
|
||
"minimum": 10,
|
||
"type": "number"
|
||
},
|
||
"strategy": {
|
||
"description": "Evaluate once the unit has had no new activity for the quiet period.",
|
||
"enum": ["inactivity"],
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": ["strategy"],
|
||
"type": "object"
|
||
}
|
||
],
|
||
"description": "Target-specific config. For 'trace' and 'session' targets: a settle config discriminated on `strategy`, either 'fixed_window' {window_seconds} or 'inactivity' {quiet_period_seconds, max_age_seconds}. Send `strategy` explicitly. The server fills in any other field you omit, using per-target defaults, and the accepted bounds also depend on `target`. Empty for 'generation'."
|
||
}
|
||
},
|
||
"required": ["name", "evaluation_type", "output_type"],
|
||
"type": "object"
|
||
}
|