mirror of
https://github.com/hanzoai/team.git
synced 2026-09-30 01:57:35 +00:00
718 lines
29 KiB
TypeScript
718 lines
29 KiB
TypeScript
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See LICENSE.txt for license information.
|
|
|
|
import {expect, test} from './ai_bridge_fixture';
|
|
import {
|
|
createChannelWithManyPosts,
|
|
createRecapAndWaitForStatus,
|
|
createUnreadChannelFixture,
|
|
markAllCurrentChannelsRead,
|
|
setupRecapBridge,
|
|
waitForRecapStatus,
|
|
waitForRecordedRequestCount,
|
|
} from './recaps_helpers';
|
|
|
|
/**
|
|
* @objective Verify a user can create a selected-channels AI recap and receive the mocked summary without reloading the page
|
|
*/
|
|
test('creates selected-channels recap and auto-renders mocked summary', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
const recapHighlight = `Deterministic highlight ${pw.random.id()}`;
|
|
const recapActionItem = `Deterministic action item ${pw.random.id()}`;
|
|
const recapTitle = `AI recap ${pw.random.id()}`;
|
|
const sourceMessage = `Please summarize this update ${pw.random.id()}`;
|
|
|
|
// # Initialize the test server state and configure one deterministic recap agent.
|
|
const {adminClient, adminUser, team, user, userClient} = await pw.initSetup();
|
|
if (!adminUser) {
|
|
throw new Error('Failed to create admin user');
|
|
}
|
|
|
|
const {agent} = await setupRecapBridge(pw, adminClient, {
|
|
completions: [
|
|
pw.recapCompletion({
|
|
highlights: [recapHighlight],
|
|
actionItems: [recapActionItem],
|
|
}),
|
|
],
|
|
});
|
|
|
|
// # Seed a real unread post so the recap job has content to summarize.
|
|
const channel = await createUnreadChannelFixture(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'AI Recap Selected Channel',
|
|
sourceMessage,
|
|
);
|
|
|
|
// # Login as the end user, create the recap through the selected-channels modal flow, and submit it.
|
|
const {recapsPage} = await pw.testBrowser.login(user);
|
|
await recapsPage.goto(team.name);
|
|
await recapsPage.toBeVisible();
|
|
|
|
const createRecapModal = await recapsPage.openCreateRecap();
|
|
await createRecapModal.fillTitle(recapTitle);
|
|
await createRecapModal.selectSelectedChannels();
|
|
await createRecapModal.enableRunOnce();
|
|
await createRecapModal.clickNext();
|
|
await createRecapModal.expectChannelSelectorVisible();
|
|
await createRecapModal.searchChannel(channel.display_name);
|
|
await createRecapModal.selectChannel(channel.display_name);
|
|
await createRecapModal.clickNext();
|
|
await createRecapModal.expectSummaryChannels([channel.display_name]);
|
|
await createRecapModal.startRecap();
|
|
|
|
// * Verify the recap first appears in its pending state.
|
|
const recap = recapsPage.getRecap(recapTitle);
|
|
await recap.toBeVisible();
|
|
await recap.expectProcessing();
|
|
|
|
// * Verify the recap completes and renders the mocked summary content without a manual page reload.
|
|
await waitForRecapStatus(pw, userClient, recapTitle, 'completed');
|
|
await expect(recap.container).toContainText(recapHighlight, {timeout: pw.duration.one_min});
|
|
await expect(recap.container).toContainText(recapActionItem, {timeout: pw.duration.one_min});
|
|
|
|
// * Verify the recorded bridge request used the recap_summary operation and included the unread source message.
|
|
await waitForRecordedRequestCount(pw, adminClient, 1);
|
|
const bridgeState = await pw.getAIBridgeMock(adminClient);
|
|
const recapRequest = bridgeState.recorded_requests.find((request) => request.operation === 'recap_summary');
|
|
|
|
expect(recapRequest).toBeDefined();
|
|
expect(recapRequest?.agent_id).toBe(agent.id);
|
|
expect(recapRequest?.messages.some((message) => message.message.includes(sourceMessage))).toBe(true);
|
|
});
|
|
|
|
/**
|
|
* @objective Verify the all-unreads recap flow skips channel selection and only summarizes unread channels
|
|
*/
|
|
test('creates all-unreads recap for only unread channels', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
const recapTitle = `Unread recap ${pw.random.id()}`;
|
|
const sourceMessageOne = `Unread source one ${pw.random.id()}`;
|
|
const sourceMessageTwo = `Unread source two ${pw.random.id()}`;
|
|
const highlightOne = `Unread highlight one ${pw.random.id()}`;
|
|
const highlightTwo = `Unread highlight two ${pw.random.id()}`;
|
|
|
|
// # Initialize the test server state, clear baseline unread state, and configure two deterministic recap completions.
|
|
const {adminClient, adminUser, team, user, userClient} = await pw.initSetup();
|
|
if (!adminUser) {
|
|
throw new Error('Failed to create admin user');
|
|
}
|
|
|
|
await markAllCurrentChannelsRead(userClient, team.id);
|
|
|
|
const {agent} = await setupRecapBridge(pw, adminClient, {
|
|
completions: [
|
|
pw.recapCompletion({
|
|
highlights: [highlightOne],
|
|
actionItems: [],
|
|
}),
|
|
pw.recapCompletion({
|
|
highlights: [highlightTwo],
|
|
actionItems: [],
|
|
}),
|
|
],
|
|
});
|
|
|
|
const firstChannel = await createUnreadChannelFixture(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Unread recap one',
|
|
sourceMessageOne,
|
|
);
|
|
const secondChannel = await createUnreadChannelFixture(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Unread recap two',
|
|
sourceMessageTwo,
|
|
);
|
|
|
|
// # Open the all-unreads modal flow and advance directly to the summary step.
|
|
const {recapsPage} = await pw.testBrowser.login(user);
|
|
await recapsPage.goto(team.name);
|
|
await recapsPage.toBeVisible();
|
|
|
|
const createRecapModal = await recapsPage.openCreateRecap();
|
|
await createRecapModal.fillTitle(recapTitle);
|
|
await createRecapModal.selectAgent(agent.displayName);
|
|
await createRecapModal.selectAllUnreads();
|
|
await createRecapModal.enableRunOnce();
|
|
await createRecapModal.clickNext();
|
|
|
|
// * Verify the all-unreads flow skips the channel selector and includes only the unread channels in the summary.
|
|
await createRecapModal.expectChannelSelectorHidden();
|
|
await createRecapModal.expectSummaryChannels([firstChannel.display_name, secondChannel.display_name]);
|
|
await createRecapModal.startRecap();
|
|
|
|
// * Verify the completed recap renders summaries for both unread channels.
|
|
const recap = recapsPage.getRecap(recapTitle);
|
|
await expect(recap.container).toContainText(highlightOne, {timeout: pw.duration.one_min});
|
|
await expect(recap.container).toContainText(highlightTwo, {timeout: pw.duration.one_min});
|
|
await expect(recap.container).toContainText(firstChannel.display_name);
|
|
await expect(recap.container).toContainText(secondChannel.display_name);
|
|
|
|
// * Verify the bridge recorded one recap_summary request per unread channel and included both source messages.
|
|
await waitForRecordedRequestCount(pw, adminClient, 2);
|
|
const bridgeState = await pw.getAIBridgeMock(adminClient);
|
|
const recapRequests = bridgeState.recorded_requests.filter((request) => request.operation === 'recap_summary');
|
|
|
|
expect(recapRequests).toHaveLength(2);
|
|
expect(
|
|
recapRequests.some((request) => request.messages.some((message) => message.message.includes(sourceMessageOne))),
|
|
).toBe(true);
|
|
expect(
|
|
recapRequests.some((request) => request.messages.some((message) => message.message.includes(sourceMessageTwo))),
|
|
).toBe(true);
|
|
});
|
|
|
|
/**
|
|
* @objective Verify recap creation is disabled when the AI bridge reports itself as unavailable
|
|
*/
|
|
test('disables recap creation when the bridge is unavailable', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
// # Initialize the test server state and configure the recap bridge as unavailable.
|
|
const {adminClient, team, user} = await pw.initSetup();
|
|
await setupRecapBridge(pw, adminClient, {
|
|
available: false,
|
|
completions: [],
|
|
});
|
|
|
|
// # Open the recaps page as the end user.
|
|
const {recapsPage} = await pw.testBrowser.login(user);
|
|
await recapsPage.goto(team.name);
|
|
await recapsPage.toBeVisible();
|
|
|
|
// * Verify the page shows the caught-up empty state and disables the Add a recap button with the expected reason.
|
|
await recapsPage.expectCaughtUpEmptyState();
|
|
await recapsPage.expectAddRecapDisabled('Agents Bridge is not enabled');
|
|
});
|
|
|
|
/**
|
|
* @objective Verify marking a recap as read moves it from the Unread tab to the Read tab
|
|
*/
|
|
test('moves a recap from unread to read when marked read', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
const recapTitle = `Read state recap ${pw.random.id()}`;
|
|
const recapHighlight = `Read state highlight ${pw.random.id()}`;
|
|
const sourceMessage = `Read state source ${pw.random.id()}`;
|
|
|
|
// # Initialize the test server state, configure the recap bridge, and seed a completed recap for the user.
|
|
const {adminClient, adminUser, team, user, userClient} = await pw.initSetup();
|
|
if (!adminUser) {
|
|
throw new Error('Failed to create admin user');
|
|
}
|
|
|
|
const {agent} = await setupRecapBridge(pw, adminClient, {
|
|
completions: [
|
|
pw.recapCompletion({
|
|
highlights: [recapHighlight],
|
|
actionItems: [],
|
|
}),
|
|
],
|
|
});
|
|
|
|
const channel = await createUnreadChannelFixture(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Read state channel',
|
|
sourceMessage,
|
|
);
|
|
await createRecapAndWaitForStatus(pw, userClient, recapTitle, [channel.id], agent.id, 'completed');
|
|
|
|
// # Open the unread recap and mark it as read from the recap card.
|
|
const {recapsPage} = await pw.testBrowser.login(user);
|
|
await recapsPage.goto(team.name);
|
|
await recapsPage.toBeVisible();
|
|
|
|
const unreadRecap = recapsPage.getRecap(recapTitle);
|
|
await unreadRecap.toBeVisible();
|
|
await unreadRecap.clickMarkRead();
|
|
|
|
// * Verify the recap disappears from the Unread tab and appears in the Read tab with the inline Mark read button removed.
|
|
await recapsPage.expectRecapNotVisible(recapTitle);
|
|
await recapsPage.switchToRead();
|
|
|
|
const readRecap = recapsPage.getRecap(recapTitle);
|
|
await readRecap.toBeVisible();
|
|
await expect(readRecap.markReadButton).not.toBeVisible();
|
|
await expect(readRecap.menuButton).toBeVisible();
|
|
});
|
|
|
|
/**
|
|
* @objective Verify deleting a recap removes it from the recaps list
|
|
*/
|
|
test('deletes a recap from the recaps page', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
const recapTitle = `Delete recap ${pw.random.id()}`;
|
|
const recapHighlight = `Delete recap highlight ${pw.random.id()}`;
|
|
const sourceMessage = `Delete recap source ${pw.random.id()}`;
|
|
|
|
// # Initialize the test server state, configure the recap bridge, and seed a completed recap for the user.
|
|
const {adminClient, adminUser, team, user, userClient} = await pw.initSetup();
|
|
if (!adminUser) {
|
|
throw new Error('Failed to create admin user');
|
|
}
|
|
|
|
const {agent} = await setupRecapBridge(pw, adminClient, {
|
|
completions: [
|
|
pw.recapCompletion({
|
|
highlights: [recapHighlight],
|
|
actionItems: [],
|
|
}),
|
|
],
|
|
});
|
|
|
|
const channel = await createUnreadChannelFixture(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Delete recap channel',
|
|
sourceMessage,
|
|
);
|
|
await createRecapAndWaitForStatus(pw, userClient, recapTitle, [channel.id], agent.id, 'completed');
|
|
|
|
// # Open the recap, trigger the delete action, and confirm the delete modal.
|
|
const {recapsPage} = await pw.testBrowser.login(user);
|
|
await recapsPage.goto(team.name);
|
|
await recapsPage.toBeVisible();
|
|
|
|
const recap = recapsPage.getRecap(recapTitle);
|
|
await recap.toBeVisible();
|
|
await recap.clickDelete();
|
|
await recapsPage.confirmDelete();
|
|
|
|
// * Verify the recap disappears from the list and the page returns to the caught-up empty state.
|
|
await recapsPage.expectRecapNotVisible(recapTitle);
|
|
await recapsPage.expectCaughtUpEmptyState();
|
|
});
|
|
|
|
/**
|
|
* @objective Verify regenerating a recap sends a new summary request and replaces the rendered summary with the latest mocked response
|
|
*/
|
|
test('regenerates a recap with a new mocked summary', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
const recapTitle = `Regenerated recap ${pw.random.id()}`;
|
|
const firstHighlight = `Original summary ${pw.random.id()}`;
|
|
const secondHighlight = `Regenerated summary ${pw.random.id()}`;
|
|
const sourceMessage = `Regenerate source ${pw.random.id()}`;
|
|
|
|
// # Initialize the test server state, queue two recap completions, and seed the first completed recap.
|
|
const {adminClient, adminUser, team, user, userClient} = await pw.initSetup();
|
|
if (!adminUser) {
|
|
throw new Error('Failed to create admin user');
|
|
}
|
|
|
|
const {agent} = await setupRecapBridge(pw, adminClient, {
|
|
completions: [
|
|
pw.recapCompletion({
|
|
highlights: [firstHighlight],
|
|
actionItems: [],
|
|
}),
|
|
pw.recapCompletion({
|
|
highlights: [secondHighlight],
|
|
actionItems: [],
|
|
}),
|
|
],
|
|
});
|
|
const originalConfig = await adminClient.getConfig();
|
|
|
|
try {
|
|
await adminClient.patchConfig({
|
|
AIRecapSettings: {
|
|
EnforceCooldown: false,
|
|
},
|
|
});
|
|
|
|
const channel = await createUnreadChannelFixture(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Regenerate recap channel',
|
|
sourceMessage,
|
|
);
|
|
await createRecapAndWaitForStatus(pw, userClient, recapTitle, [channel.id], agent.id, 'completed');
|
|
|
|
// # Open the recap, confirm the original summary is visible, and trigger regeneration from the recap menu.
|
|
const {recapsPage} = await pw.testBrowser.login(user);
|
|
await recapsPage.goto(team.name);
|
|
await recapsPage.toBeVisible();
|
|
|
|
const recap = recapsPage.getRecap(recapTitle);
|
|
await recap.expand();
|
|
await recap.expectText(firstHighlight);
|
|
await recap.openMenuAction('Regenerate this recap');
|
|
|
|
// * Verify the regeneration request is recorded and then renders the regenerated summary.
|
|
await waitForRecordedRequestCount(pw, adminClient, 2);
|
|
await expect(recap.container).toContainText(secondHighlight, {timeout: pw.duration.one_min});
|
|
await expect(recap.container).not.toContainText(firstHighlight);
|
|
|
|
// * Verify two recap_summary requests were recorded for the original generation and the regeneration.
|
|
const bridgeState = await pw.getAIBridgeMock(adminClient);
|
|
const recapRequests = bridgeState.recorded_requests.filter((request) => request.operation === 'recap_summary');
|
|
expect(recapRequests).toHaveLength(2);
|
|
} finally {
|
|
await adminClient.updateConfig(originalConfig as any);
|
|
}
|
|
});
|
|
|
|
/**
|
|
* @objective Verify a failed recap renders the failed state and can recover after regeneration
|
|
*/
|
|
test('recovers a failed recap through regeneration', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
const recapTitle = `Failed recap ${pw.random.id()}`;
|
|
const recoveredHighlight = `Recovered highlight ${pw.random.id()}`;
|
|
const sourceMessage = `Failed recap source ${pw.random.id()}`;
|
|
|
|
// # Initialize the test server state, queue a failing completion followed by a successful completion, and seed the failed recap.
|
|
const {adminClient, adminUser, team, user, userClient} = await pw.initSetup();
|
|
if (!adminUser) {
|
|
throw new Error('Failed to create admin user');
|
|
}
|
|
|
|
const {agent} = await setupRecapBridge(pw, adminClient, {
|
|
completions: [
|
|
{error: 'deterministic recap failure', status_code: 500},
|
|
pw.recapCompletion({
|
|
highlights: [recoveredHighlight],
|
|
actionItems: [],
|
|
}),
|
|
],
|
|
});
|
|
|
|
const channel = await createUnreadChannelFixture(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Failed recap channel',
|
|
sourceMessage,
|
|
);
|
|
await createRecapAndWaitForStatus(pw, userClient, recapTitle, [channel.id], agent.id, 'failed');
|
|
|
|
// # Open the failed recap and trigger regeneration from the recap menu.
|
|
const {recapsPage} = await pw.testBrowser.login(user);
|
|
await recapsPage.goto(team.name);
|
|
await recapsPage.toBeVisible();
|
|
|
|
const failedRecap = recapsPage.getRecap(recapTitle);
|
|
await failedRecap.toBeVisible();
|
|
await failedRecap.expectFailed();
|
|
await expect(failedRecap.markReadButton).not.toBeVisible();
|
|
await failedRecap.openMenuAction('Regenerate this recap');
|
|
|
|
// * Verify the failed recap returns to processing and then recovers into a completed recap.
|
|
await failedRecap.expectProcessing();
|
|
await waitForRecapStatus(pw, userClient, recapTitle, 'completed');
|
|
await recapsPage.page.reload();
|
|
await recapsPage.toBeVisible();
|
|
await failedRecap.expand();
|
|
await expect(failedRecap.container).toContainText(recoveredHighlight, {timeout: pw.duration.one_min});
|
|
});
|
|
|
|
/**
|
|
* @objective Verify recap channel card actions can mark a source channel as read and navigate back into that channel
|
|
*/
|
|
test('executes recap channel card actions', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
const recapTitle = `Channel actions recap ${pw.random.id()}`;
|
|
const recapHighlight = `Channel action highlight ${pw.random.id()}`;
|
|
const sourceMessage = `Channel action source ${pw.random.id()}`;
|
|
|
|
// # Initialize the test server state, configure the recap bridge, and seed a completed recap with one unread channel.
|
|
const {adminClient, adminUser, team, user, userClient} = await pw.initSetup();
|
|
if (!adminUser) {
|
|
throw new Error('Failed to create admin user');
|
|
}
|
|
|
|
const {agent} = await setupRecapBridge(pw, adminClient, {
|
|
completions: [
|
|
pw.recapCompletion({
|
|
highlights: [recapHighlight],
|
|
actionItems: [],
|
|
}),
|
|
],
|
|
});
|
|
|
|
const channel = await createUnreadChannelFixture(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Channel action recap',
|
|
sourceMessage,
|
|
);
|
|
await createRecapAndWaitForStatus(pw, userClient, recapTitle, [channel.id], agent.id, 'completed');
|
|
|
|
// # Open the recap, mark the recap channel as read from the channel card, and then navigate back into the source channel.
|
|
const {page, recapsPage} = await pw.testBrowser.login(user);
|
|
await recapsPage.goto(team.name);
|
|
await recapsPage.toBeVisible();
|
|
|
|
const recap = recapsPage.getRecap(recapTitle);
|
|
await recap.expand();
|
|
|
|
const recapChannelCard = recap.getChannelCard(channel.display_name);
|
|
await recapChannelCard.toBeVisible();
|
|
await recapChannelCard.openMenuAction('Mark this channel as read');
|
|
|
|
// * Verify the recap channel read action clears the unread count for the summarized channel.
|
|
await pw.waitUntil(
|
|
async () => {
|
|
const channelMember = await userClient.getMyChannelMember(channel.id);
|
|
return channelMember.mention_count === 0;
|
|
},
|
|
{timeout: pw.duration.one_min},
|
|
);
|
|
|
|
// # Use the channel card to navigate back into the original channel.
|
|
await recapChannelCard.clickChannelName();
|
|
|
|
// * Verify the user lands back in the source channel route.
|
|
await expect(page).toHaveURL(new RegExp(`/${team.name}/channels/${channel.name}$`));
|
|
});
|
|
|
|
/**
|
|
* @objective Verify the server enforces MaxTokensPerRecap by trimming source posts before summarizing,
|
|
* and includes all posts when token enforcement is disabled
|
|
*/
|
|
test('enforces the per-recap token limit on the immediate recap path', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
const limitedTitle = `Token limited recap ${pw.random.id()}`;
|
|
const unlimitedTitle = `Token unlimited recap ${pw.random.id()}`;
|
|
const highlight = `Token limit highlight ${pw.random.id()}`;
|
|
|
|
// # Initialize the test server state and queue one recap completion per immediate recap run.
|
|
const {adminClient, adminUser, team, user, userClient} = await pw.initSetup();
|
|
if (!adminUser) {
|
|
throw new Error('Failed to create admin user');
|
|
}
|
|
|
|
const {agent} = await setupRecapBridge(pw, adminClient, {
|
|
completions: [
|
|
pw.recapCompletion({highlights: [highlight], actionItems: []}),
|
|
pw.recapCompletion({highlights: [highlight], actionItems: []}),
|
|
],
|
|
});
|
|
|
|
const originalConfig = await adminClient.getConfig();
|
|
|
|
// Each post is padded to ~400 characters (~100 estimated tokens at 4 chars/token), so a 150-token
|
|
// budget admits only the single newest post once enforcement is on.
|
|
const postCount = 10;
|
|
const messageLength = 400;
|
|
|
|
try {
|
|
// # Enforce a small per-recap token budget and disable the cooldown so two recaps can run back to back.
|
|
await adminClient.patchConfig({
|
|
AIRecapSettings: {
|
|
EnforceCooldown: false,
|
|
EnforceTokensPerRecap: true,
|
|
DefaultLimits: {
|
|
MaxTokensPerRecap: 150,
|
|
},
|
|
},
|
|
});
|
|
|
|
const {channel, markers} = await createChannelWithManyPosts(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Token limit channel',
|
|
postCount,
|
|
messageLength,
|
|
);
|
|
|
|
// # Run an immediate recap with token enforcement ON.
|
|
const limitedRecap = await createRecapAndWaitForStatus(
|
|
pw,
|
|
userClient,
|
|
limitedTitle,
|
|
[channel.id],
|
|
agent.id,
|
|
'completed',
|
|
);
|
|
|
|
// * Verify the server trimmed the source posts: fewer than were created are recorded on the recap channel.
|
|
const limitedChannel = limitedRecap.channels?.find((recapChannel) => recapChannel.channel_id === channel.id);
|
|
expect(limitedChannel).toBeDefined();
|
|
const limitedCount = limitedChannel?.source_post_ids.length ?? 0;
|
|
expect(limitedCount).toBeGreaterThan(0);
|
|
expect(limitedCount).toBeLessThan(postCount);
|
|
|
|
// # Disable token enforcement and run a second immediate recap over the same channel.
|
|
await adminClient.patchConfig({
|
|
AIRecapSettings: {
|
|
EnforceCooldown: false,
|
|
EnforceTokensPerRecap: false,
|
|
},
|
|
});
|
|
|
|
const unlimitedRecap = await createRecapAndWaitForStatus(
|
|
pw,
|
|
userClient,
|
|
unlimitedTitle,
|
|
[channel.id],
|
|
agent.id,
|
|
'completed',
|
|
);
|
|
|
|
// * Verify all created posts are recorded when enforcement is off, and that it exceeds the trimmed count.
|
|
const unlimitedChannel = unlimitedRecap.channels?.find(
|
|
(recapChannel) => recapChannel.channel_id === channel.id,
|
|
);
|
|
expect(unlimitedChannel).toBeDefined();
|
|
const unlimitedCount = unlimitedChannel?.source_post_ids.length ?? 0;
|
|
expect(unlimitedCount).toBeGreaterThanOrEqual(postCount);
|
|
expect(limitedCount).toBeLessThan(unlimitedCount);
|
|
|
|
// * Verify the recorded LLM payloads corroborate the trimming: fewer seeded posts reached the bridge
|
|
// for the enforced recap than for the unenforced one.
|
|
await waitForRecordedRequestCount(pw, adminClient, 2);
|
|
const bridgeState = await pw.getAIBridgeMock(adminClient);
|
|
const recapRequests = bridgeState.recorded_requests.filter((request) => request.operation === 'recap_summary');
|
|
expect(recapRequests).toHaveLength(2);
|
|
|
|
const countMarkersInRequest = (request: (typeof recapRequests)[number]) => {
|
|
const text = request.messages.map((message) => message.message).join('\n');
|
|
return markers.filter((marker) => text.includes(marker)).length;
|
|
};
|
|
|
|
const limitedMarkers = countMarkersInRequest(recapRequests[0]);
|
|
const unlimitedMarkers = countMarkersInRequest(recapRequests[1]);
|
|
expect(limitedMarkers).toBeGreaterThan(0);
|
|
expect(limitedMarkers).toBeLessThan(postCount);
|
|
expect(unlimitedMarkers).toBe(postCount);
|
|
expect(limitedMarkers).toBeLessThan(unlimitedMarkers);
|
|
} finally {
|
|
await adminClient.updateConfig(originalConfig as any);
|
|
}
|
|
});
|
|
|
|
/**
|
|
* @objective Verify MaxTokensPerRecap is enforced as an independent per-channel cap: a single recap
|
|
* spanning two channels trims each channel's source posts to its own budget rather than sharing one
|
|
* cross-channel budget
|
|
*/
|
|
test('enforces the per-recap token limit independently per channel', {tag: '@ai_recaps'}, async ({pw}) => {
|
|
const recapTitle = `Per-channel token recap ${pw.random.id()}`;
|
|
const highlight = `Per-channel token highlight ${pw.random.id()}`;
|
|
|
|
// # Initialize the test server state and queue one recap completion per channel summarized.
|
|
const {adminClient, adminUser, team, user, userClient} = await pw.initSetup();
|
|
if (!adminUser) {
|
|
throw new Error('Failed to create admin user');
|
|
}
|
|
|
|
const {agent} = await setupRecapBridge(pw, adminClient, {
|
|
completions: [
|
|
pw.recapCompletion({highlights: [highlight], actionItems: []}),
|
|
pw.recapCompletion({highlights: [highlight], actionItems: []}),
|
|
],
|
|
});
|
|
|
|
const originalConfig = await adminClient.getConfig();
|
|
|
|
// Each post is padded to ~400 characters (~100 estimated tokens), so a 150-token per-channel
|
|
// budget admits only the single newest post in each channel once enforcement is on.
|
|
const postCount = 6;
|
|
const messageLength = 400;
|
|
|
|
try {
|
|
// # Enforce a small per-channel token budget; disable cooldown so the immediate recap runs cleanly.
|
|
await adminClient.patchConfig({
|
|
AIRecapSettings: {
|
|
EnforceCooldown: false,
|
|
EnforceTokensPerRecap: true,
|
|
DefaultLimits: {
|
|
MaxTokensPerRecap: 150,
|
|
},
|
|
},
|
|
});
|
|
|
|
const first = await createChannelWithManyPosts(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Per-channel token channel one',
|
|
postCount,
|
|
messageLength,
|
|
);
|
|
const second = await createChannelWithManyPosts(
|
|
pw,
|
|
adminClient,
|
|
adminUser.id,
|
|
user.id,
|
|
team.id,
|
|
'Per-channel token channel two',
|
|
postCount,
|
|
messageLength,
|
|
);
|
|
|
|
// # Run a single immediate recap covering both seeded channels.
|
|
const recap = await createRecapAndWaitForStatus(
|
|
pw,
|
|
userClient,
|
|
recapTitle,
|
|
[first.channel.id, second.channel.id],
|
|
agent.id,
|
|
'completed',
|
|
);
|
|
|
|
// * Verify each channel was trimmed independently to its own per-channel budget.
|
|
const firstChannel = recap.channels?.find((recapChannel) => recapChannel.channel_id === first.channel.id);
|
|
const secondChannel = recap.channels?.find((recapChannel) => recapChannel.channel_id === second.channel.id);
|
|
expect(firstChannel).toBeDefined();
|
|
expect(secondChannel).toBeDefined();
|
|
|
|
const firstCount = firstChannel?.source_post_ids.length ?? 0;
|
|
const secondCount = secondChannel?.source_post_ids.length ?? 0;
|
|
|
|
expect(firstCount).toBeGreaterThan(0);
|
|
expect(firstCount).toBeLessThan(postCount);
|
|
expect(secondCount).toBeGreaterThan(0);
|
|
expect(secondCount).toBeLessThan(postCount);
|
|
|
|
// * Verify the cap is per channel, not a shared cross-channel budget: a shared 150-token budget
|
|
// could only retain posts in one channel (exhausting it before the second), so both channels
|
|
// retaining posts means the combined kept count exceeds any single channel's budget.
|
|
expect(firstCount + secondCount).toBeGreaterThan(firstCount);
|
|
expect(firstCount + secondCount).toBeGreaterThan(secondCount);
|
|
|
|
// * Verify the recorded LLM payloads corroborate per-channel trimming: one request per channel,
|
|
// each carrying only its own trimmed subset of seeded markers.
|
|
await waitForRecordedRequestCount(pw, adminClient, 2);
|
|
const bridgeState = await pw.getAIBridgeMock(adminClient);
|
|
const recapRequests = bridgeState.recorded_requests.filter((request) => request.operation === 'recap_summary');
|
|
expect(recapRequests).toHaveLength(2);
|
|
|
|
const countMarkers = (markers: string[]) => {
|
|
const text = recapRequests
|
|
.flatMap((request) => request.messages.map((message) => message.message))
|
|
.join('\n');
|
|
return markers.filter((marker) => text.includes(marker)).length;
|
|
};
|
|
|
|
const firstMarkers = countMarkers(first.markers);
|
|
const secondMarkers = countMarkers(second.markers);
|
|
expect(firstMarkers).toBe(firstCount);
|
|
expect(secondMarkers).toBe(secondCount);
|
|
} finally {
|
|
await adminClient.updateConfig(originalConfig as any);
|
|
}
|
|
});
|