mirror of
https://github.com/hanzoai/insights.git
synced 2026-10-30 16:15:31 +00:00
zeekay
34073201e6
There were two. Ours — AI_GATEWAY_URL, addressed with the caller's IAM token, so the org that pays is read off the bearer's claim. And upstream's, addressed with LLM_GATEWAY_URL and a single shared personal key, with the product in the path. Neither of the two was working. The upstream pair is unset in production, so the 27 call sites that reach for it — signals, conversations, growth, stamp, review, warehouse enrichment, path cleaning, AI observability translation — raised "LLM_GATEWAY_URL and an API key must be configured" and had never run. And the three build_* functions that were supposed to prefer ours referenced names that do not exist in their scope (`user`, `api_key`, `bearer`), so they raised NameError before reaching either gateway. Every AI feature in this repo was down, in one of two ways, and the fallback hid which. So there is one now. The three get_* constructors keep their names, because 27 files call them, and resolve to the Hanzo gateway; the three build_* variants differ only in tagging the generation with an explicit ai_product. Both go through one resolver, and every label — ai_product, ai_stage, team_id, the trace id — is built in one place rather than by each caller. A static key is deleted rather than moved. A key in the environment is one tenant's credential doing every tenant's work: it would bill every customer's assistant to whoever owns the key. The bearer is an IAM token and nothing else — a signed-in person's own token when someone is acting, this deployment's identity when nobody is. And there is no fallback. Unset, malformed, or IAM refusing a token now raises GatewayUnavailable. Answering anyway would mean answering from outside the estate, unbilled and unattributed, which is worse than not answering. Verified against the live Django with the real SDKs: 20 assertions on identity, base URLs, labels and refusal, and all 23 calling modules import.
106 lines
3.7 KiB
Python
106 lines
3.7 KiB
Python
import os
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from unittest.mock import patch
|
|
|
|
from django.conf import settings
|
|
|
|
import hanzo_insights
|
|
from dotenv import load_dotenv
|
|
from hanzo_insights import Insights
|
|
from hanzo_insights.ai.openai import AsyncOpenAI
|
|
|
|
from insights.llm.gateway_client import get_async_anthropic_gateway_client
|
|
from insights.models import Organization, Team
|
|
|
|
# This eval runs via bare pytest (no insightscli), so the conftest owns loading repo-root .env.
|
|
load_dotenv(Path(__file__).resolve().parents[3] / ".env")
|
|
|
|
# Initialize hanzo_insights default_client so the LLM wrapper (which requires it) works
|
|
hanzo_insights.default_client = Insights( # ty: ignore[invalid-assignment]
|
|
os.environ.get("INSIGHTS_PROJECT_API_KEY", "sk-unused"),
|
|
host=os.environ.get("INSIGHTS_HOST", "http://localhost:8010"),
|
|
disabled=True,
|
|
debug=bool(os.environ.get("INSIGHTS_DEBUG")),
|
|
)
|
|
|
|
# Django settings are loaded before conftest, so .env vars aren't picked up.
|
|
# Override settings that need to come from .env.
|
|
if not getattr(settings, "OPENAI_API_KEY", ""):
|
|
settings.OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "")
|
|
if not getattr(settings, "AI_GATEWAY_URL", ""):
|
|
settings.AI_GATEWAY_URL = os.environ.get("AI_GATEWAY_URL", "")
|
|
|
|
# Team id used by the eval harness when attributing LLM cost via the gateway.
|
|
EVAL_TEAM_ID = int(os.environ.get("SIGNALS_EVAL_TEAM_ID", "1"))
|
|
|
|
|
|
def pytest_addoption(parser):
|
|
parser.addoption("--limit", default=None, type=int, help="Limit number of items to process (e.g. --limit 3)")
|
|
parser.addoption("--no-capture", action="store_true", default=False, help="Skip emitting eval results to Insights")
|
|
parser.addoption("--online", action="store_true", default=False, help="Capture as online eval")
|
|
parser.addoption("--strategy", default="example_strategy", help="Strategy module name (default: example_strategy)")
|
|
parser.addoption(
|
|
"--safe", action="store_true", default=False, help="Filter to safe signals only, skip safety_filter step"
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def limit(request):
|
|
return request.config.getoption("--limit")
|
|
|
|
|
|
@pytest.fixture
|
|
def no_capture(request):
|
|
return request.config.getoption("--no-capture")
|
|
|
|
|
|
@pytest.fixture
|
|
def online(request):
|
|
return request.config.getoption("--online")
|
|
|
|
|
|
@pytest.fixture
|
|
def insights_client(no_capture, db):
|
|
api_key = os.environ.get("INSIGHTS_PROJECT_API_KEY", "")
|
|
if not api_key:
|
|
last_team = Team.objects.order_by("-pk").first()
|
|
if last_team:
|
|
api_key = last_team.api_token
|
|
elif not no_capture:
|
|
org = Organization.objects.create(name="Eval Org", is_ai_data_processing_approved=True)
|
|
team = Team.objects.create(organization=org, name="Eval Team")
|
|
api_key = team.api_token
|
|
if not api_key and not no_capture:
|
|
raise ValueError("INSIGHTS_PROJECT_API_KEY needs to be set (or pass --no-capture).")
|
|
host = os.environ.get("INSIGHTS_HOST", "http://localhost:8010")
|
|
client = Insights(
|
|
api_key or "sk-unused", host=host, disabled=no_capture, debug=bool(os.environ.get("INSIGHTS_DEBUG"))
|
|
)
|
|
yield client
|
|
client.shutdown()
|
|
|
|
|
|
@pytest.fixture
|
|
async def openai_client(insights_client):
|
|
client = AsyncOpenAI(insights_client=insights_client)
|
|
yield client
|
|
await client.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def gateway_client():
|
|
"""Async Anthropic client pointed at the internal LLM gateway's native Messages endpoint.
|
|
|
|
Used by eval_grouping_e2e to drive the production signals pre-emit pipeline
|
|
(`_check_actionability` etc.) through the gateway, attributing cost to EVAL_TEAM_ID.
|
|
"""
|
|
return get_async_anthropic_gateway_client(product="signals", team_id=EVAL_TEAM_ID)
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_temporal():
|
|
with patch("temporalio.activity.heartbeat"):
|
|
yield
|