mirror of
https://github.com/hanzoai/insights.git
synced 2026-10-30 16:15:31 +00:00
zeekay
40d120dbf7
Every first-party Python caller and test now names the mount the API actually answers on. Mechanical: URL literals only, and only where the /api/ was ours -- a preceding word character means it was a module path like insights/api/ or a third-party host like slack.com/api/, and those are untouched. Left saying /api/, each for a reason rather than for safety: - products/warehouse_sources/.../sources/** is a catalogue of vendor endpoints - services/llm-gateway deploys separately and is under a code freeze, so moving its two calls would break it during the skew - the mcp_store and Slack/Teams redirect_uris are registered with third parties and must byte-match what was registered - the Wizard and Raycast client-metadata paths are OAuth client_ids - /api/surveys and /api/feature_flag/local_evaluation are SDK aliases the ingress routes to the Rust fleets, held by released SDK versions Test URLconfs moved with the requests that call them, and the Datastore user-selection fixtures now carry the leading slash that request.path has.
459 lines
18 KiB
TypeScript
459 lines
18 KiB
TypeScript
import { expectLogic } from 'kea-test-utils'
|
|
|
|
import { useMocks } from '~/mocks/jest'
|
|
import { initKeaTests } from '~/test/init'
|
|
|
|
import { filterGroups, findSelectedProvider, parsePlaygroundProviderKeyId } from './ModelPicker'
|
|
import {
|
|
buildPlaygroundProviderModelGroups,
|
|
modelPickerLogic,
|
|
type ModelOption,
|
|
type ProviderModelGroup,
|
|
} from './modelPickerLogic'
|
|
|
|
// API responses use snake_case is_recommended
|
|
const BYOK_OPENAI_MODELS = [
|
|
{ id: 'gpt-4.1', name: 'GPT-4.1', provider: 'OpenAI', description: '', is_recommended: true },
|
|
{ id: 'gpt-5', name: 'GPT-5', provider: 'OpenAI', description: '', is_recommended: true },
|
|
]
|
|
|
|
const BYOK_OPENAI_MODELS_MIXED = [
|
|
{ id: 'gpt-4.1', name: 'GPT-4.1', provider: 'OpenAI', description: '', is_recommended: true },
|
|
{ id: 'gpt-5', name: 'GPT-5', provider: 'OpenAI', description: '', is_recommended: true },
|
|
{
|
|
id: 'gpt-4o-audio-preview',
|
|
name: 'gpt-4o-audio-preview',
|
|
provider: 'OpenAI',
|
|
description: '',
|
|
is_recommended: false,
|
|
},
|
|
]
|
|
|
|
const BYOK_ANTHROPIC_MODELS = [
|
|
{ id: 'claude-sonnet-4', name: 'Claude Sonnet 4', provider: 'Anthropic', description: '', is_recommended: true },
|
|
]
|
|
|
|
describe('modelPickerLogic', () => {
|
|
let logic: ReturnType<typeof modelPickerLogic.build>
|
|
|
|
beforeEach(() => {
|
|
initKeaTests()
|
|
})
|
|
|
|
afterEach(() => {
|
|
logic?.unmount()
|
|
})
|
|
|
|
describe('loadByokModels', () => {
|
|
it('should load and attach providerKeyId to models from valid keys', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [{ id: 'key-1', provider: 'openai', state: 'ok' }],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
'/v1/llm_proxy/models/': ({ request }) => {
|
|
if (new URL(request.url).searchParams.get('provider_key_id') === 'key-1') {
|
|
return [200, BYOK_OPENAI_MODELS]
|
|
}
|
|
return [200, []]
|
|
},
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
expect(logic.values.byokModels).toEqual(
|
|
BYOK_OPENAI_MODELS.map((m) => ({
|
|
id: m.id,
|
|
name: m.name,
|
|
provider: m.provider,
|
|
description: m.description,
|
|
isRecommended: true,
|
|
providerKeyId: 'key-1',
|
|
}))
|
|
)
|
|
})
|
|
|
|
it('should map is_recommended correctly for both true and false values', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [{ id: 'key-1', provider: 'openai', state: 'ok' }],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
'/v1/llm_proxy/models/': ({ request }) => {
|
|
if (new URL(request.url).searchParams.get('provider_key_id') === 'key-1') {
|
|
return [200, BYOK_OPENAI_MODELS_MIXED]
|
|
}
|
|
return [200, []]
|
|
},
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
const models = logic.values.byokModels
|
|
expect(models).toHaveLength(3)
|
|
|
|
const recommended = models.filter((m) => m.isRecommended)
|
|
const nonRecommended = models.filter((m) => !m.isRecommended)
|
|
expect(recommended).toHaveLength(2)
|
|
expect(nonRecommended).toHaveLength(1)
|
|
expect(nonRecommended[0].id).toBe('gpt-4o-audio-preview')
|
|
})
|
|
|
|
it('should return empty array when no valid keys exist', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/llm_proxy/models/': () => [200, []],
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [{ id: 'key-1', provider: 'openai', state: 'invalid' }],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
expect(logic.values.byokModels).toEqual([])
|
|
})
|
|
|
|
it('should deduplicate models by providerKeyId and model id', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [
|
|
{ id: 'key-1', provider: 'openai', state: 'ok' },
|
|
{ id: 'key-2', provider: 'anthropic', state: 'ok' },
|
|
],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
'/v1/llm_proxy/models/': ({ request }) => {
|
|
const keyId = new URL(request.url).searchParams.get('provider_key_id')
|
|
if (keyId === 'key-1') {
|
|
return [200, BYOK_OPENAI_MODELS]
|
|
}
|
|
if (keyId === 'key-2') {
|
|
return [200, BYOK_ANTHROPIC_MODELS]
|
|
}
|
|
return [200, []]
|
|
},
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
const openaiModels = logic.values.byokModels.filter((m) => m.providerKeyId === 'key-1')
|
|
const anthropicModels = logic.values.byokModels.filter((m) => m.providerKeyId === 'key-2')
|
|
expect(openaiModels).toHaveLength(2)
|
|
expect(anthropicModels).toHaveLength(1)
|
|
})
|
|
|
|
it('should gracefully handle API errors for individual keys', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [
|
|
{ id: 'key-1', provider: 'openai', state: 'ok' },
|
|
{ id: 'key-2', provider: 'anthropic', state: 'ok' },
|
|
],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
'/v1/llm_proxy/models/': ({ request }) => {
|
|
const keyId = new URL(request.url).searchParams.get('provider_key_id')
|
|
if (keyId === 'key-1') {
|
|
return [500, { error: 'Internal error' }]
|
|
}
|
|
if (keyId === 'key-2') {
|
|
return [200, BYOK_ANTHROPIC_MODELS]
|
|
}
|
|
return [200, []]
|
|
},
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
// key-1 failed, but key-2 models should still be present
|
|
expect(logic.values.byokModels).toEqual(
|
|
BYOK_ANTHROPIC_MODELS.map((m) => ({
|
|
id: m.id,
|
|
name: m.name,
|
|
provider: m.provider,
|
|
description: m.description,
|
|
isRecommended: true,
|
|
providerKeyId: 'key-2',
|
|
}))
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('hasByokKeys', () => {
|
|
it('should return true when at least one key has ok state', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [
|
|
{ id: 'key-1', provider: 'openai', state: 'invalid' },
|
|
{ id: 'key-2', provider: 'anthropic', state: 'ok' },
|
|
],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
'/v1/llm_proxy/models/': () => [200, []],
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
expect(logic.values.hasByokKeys).toBe(true)
|
|
})
|
|
|
|
it('should return false when all keys are non-ok', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/llm_proxy/models/': () => [200, []],
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [
|
|
{ id: 'key-1', provider: 'openai', state: 'invalid' },
|
|
{ id: 'key-2', provider: 'anthropic', state: 'error' },
|
|
],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
expect(logic.values.hasByokKeys).toBe(false)
|
|
})
|
|
|
|
it('should return false when no keys exist', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/llm_proxy/models/': () => [200, []],
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
expect(logic.values.hasByokKeys).toBe(false)
|
|
})
|
|
})
|
|
|
|
describe('providerModelGroups', () => {
|
|
it('should group models by provider key', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [
|
|
{ id: 'key-1', provider: 'openai', name: 'My OpenAI Key', state: 'ok' },
|
|
{ id: 'key-2', provider: 'anthropic', name: 'My Anthropic Key', state: 'ok' },
|
|
],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
'/v1/llm_proxy/models/': ({ request }) => {
|
|
const keyId = new URL(request.url).searchParams.get('provider_key_id')
|
|
if (keyId === 'key-1') {
|
|
return [200, BYOK_OPENAI_MODELS]
|
|
}
|
|
if (keyId === 'key-2') {
|
|
return [200, BYOK_ANTHROPIC_MODELS]
|
|
}
|
|
return [200, []]
|
|
},
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
const groups = logic.values.providerModelGroups
|
|
expect(groups).toHaveLength(2)
|
|
|
|
const labels = groups.map((g) => g.label).sort()
|
|
expect(labels).toEqual(['Anthropic', 'OpenAI'])
|
|
|
|
const openaiGroup = groups.find((g) => g.provider === 'openai')
|
|
expect(openaiGroup?.models).toHaveLength(2)
|
|
expect(openaiGroup?.providerKeyId).toBe('key-1')
|
|
})
|
|
|
|
it('should disambiguate labels when multiple keys exist for same provider', async () => {
|
|
useMocks({
|
|
get: {
|
|
'/v1/environments/:team_id/llm_analytics/provider_keys/': {
|
|
results: [
|
|
{ id: 'key-1', provider: 'openai', name: 'Production', state: 'ok' },
|
|
{ id: 'key-2', provider: 'openai', name: 'Staging', state: 'ok' },
|
|
],
|
|
},
|
|
'/v1/environments/:team_id/llm_analytics/evaluation_config/': {
|
|
active_provider_key: null,
|
|
},
|
|
'/v1/llm_proxy/models/': ({ request }) => {
|
|
const keyId = new URL(request.url).searchParams.get('provider_key_id')
|
|
if (keyId === 'key-1' || keyId === 'key-2') {
|
|
return [200, BYOK_OPENAI_MODELS]
|
|
}
|
|
return [200, []]
|
|
},
|
|
},
|
|
})
|
|
|
|
logic = modelPickerLogic()
|
|
logic.mount()
|
|
await expectLogic(logic).toFinishAllListeners()
|
|
|
|
const groups = logic.values.providerModelGroups
|
|
expect(groups).toHaveLength(2)
|
|
|
|
const labels = groups.map((g) => g.label).sort()
|
|
expect(labels).toEqual(['OpenAI (Production)', 'OpenAI (Staging)'])
|
|
})
|
|
})
|
|
})
|
|
|
|
const MODEL_A: ModelOption = { id: 'gpt-5', name: 'GPT-5', provider: 'OpenAI', description: '' }
|
|
const MODEL_B: ModelOption = { id: 'claude-sonnet-4', name: 'Claude Sonnet 4', provider: 'Anthropic', description: '' }
|
|
const MODEL_C: ModelOption = {
|
|
id: 'gpt-4.1',
|
|
name: 'GPT-4.1',
|
|
provider: 'OpenAI',
|
|
description: '',
|
|
isRecommended: true,
|
|
}
|
|
|
|
const GROUPS: ProviderModelGroup[] = [
|
|
{ provider: 'openai', providerKeyId: 'key-1', label: 'OpenAI', models: [MODEL_A, MODEL_C] },
|
|
{ provider: 'anthropic', providerKeyId: 'key-2', label: 'Anthropic', models: [MODEL_B] },
|
|
]
|
|
|
|
describe('parsePlaygroundProviderKeyId', () => {
|
|
// The 'playground:' case exercises the unknown-provider path, which logs a console.error by design.
|
|
let consoleErrorSpy: jest.SpyInstance
|
|
|
|
beforeEach(() => {
|
|
consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation()
|
|
})
|
|
|
|
afterEach(() => {
|
|
consoleErrorSpy.mockRestore()
|
|
})
|
|
|
|
it.each([
|
|
['playground:openai', 'openai'],
|
|
['playground:anthropic', 'anthropic'],
|
|
['key-123', null],
|
|
['', null],
|
|
['playground:', null],
|
|
])('parsePlaygroundProviderKeyId(%s) => %s', (input, expected) => {
|
|
expect(parsePlaygroundProviderKeyId(input)).toBe(expected)
|
|
if (input === 'playground:') {
|
|
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Unknown LLM provider'))
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('findSelectedProvider', () => {
|
|
it.each([
|
|
['exact match by providerKeyId and model', 'gpt-5', 'key-1', 'openai'],
|
|
['fallback to model-only match when providerKeyId is null', 'claude-sonnet-4', null, 'anthropic'],
|
|
['returns null when model not found', 'unknown-model', 'key-1', null],
|
|
['returns null for empty groups', 'gpt-5', 'key-1', null],
|
|
] as const)('%s', (_, model, providerKeyId, expected) => {
|
|
const groups = expected === null && model === 'gpt-5' ? [] : GROUPS
|
|
expect(findSelectedProvider(groups as ProviderModelGroup[], model, providerKeyId)).toBe(expected)
|
|
})
|
|
})
|
|
|
|
describe('filterGroups', () => {
|
|
it('returns all groups when search is empty', () => {
|
|
expect(filterGroups(GROUPS, '')).toBe(GROUPS)
|
|
})
|
|
|
|
it('filters models by name (case-insensitive)', () => {
|
|
const result = filterGroups(GROUPS, 'claude')
|
|
expect(result).toHaveLength(1)
|
|
expect(result[0].provider).toBe('anthropic')
|
|
})
|
|
|
|
it('filters models by id', () => {
|
|
const result = filterGroups(GROUPS, 'gpt-4.1')
|
|
expect(result).toHaveLength(1)
|
|
expect(result[0].models).toHaveLength(1)
|
|
expect(result[0].models[0].id).toBe('gpt-4.1')
|
|
})
|
|
|
|
it('removes groups with no matching models', () => {
|
|
expect(filterGroups(GROUPS, 'nonexistent')).toHaveLength(0)
|
|
})
|
|
})
|
|
|
|
describe('buildPlaygroundProviderModelGroups', () => {
|
|
it('groups models by provider with playground: prefix keys', () => {
|
|
const models: ModelOption[] = [MODEL_A, MODEL_C, MODEL_B]
|
|
const groups = buildPlaygroundProviderModelGroups(models)
|
|
|
|
expect(groups).toHaveLength(2)
|
|
expect(groups[0].provider).toBe('openai')
|
|
expect(groups[0].providerKeyId).toBe('playground:openai')
|
|
expect(groups[0].models).toHaveLength(2)
|
|
expect(groups[1].provider).toBe('anthropic')
|
|
expect(groups[1].providerKeyId).toBe('playground:anthropic')
|
|
expect(groups[1].models).toHaveLength(1)
|
|
})
|
|
|
|
it('returns empty array for empty input', () => {
|
|
expect(buildPlaygroundProviderModelGroups([])).toEqual([])
|
|
})
|
|
|
|
it('sorts groups by provider order', () => {
|
|
const models: ModelOption[] = [
|
|
{ ...MODEL_B, provider: 'Anthropic' },
|
|
{ ...MODEL_A, provider: 'OpenAI' },
|
|
]
|
|
const groups = buildPlaygroundProviderModelGroups(models)
|
|
expect(groups.map((g) => g.provider)).toEqual(['openai', 'anthropic'])
|
|
})
|
|
})
|