mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
test(desktop): cover correction resume without duplicate prompts (#69708)
* test(desktop): cover correction resume without duplicate prompts Exercise a live composer correction, switch away and back before the response settles, and assert both user turns retain their order and occur exactly once. * test(desktop): cover correction warm resume during tool run Exercise a correction accepted at a foreground-tool boundary, a switch through a persisted session, and the warm resume back. Assert the original prompt and correction remain singular and ordered.
This commit is contained in:
ethernet
GitHub
parent
deadb43cc2
commit
a23e39fe6d
2 changed files with 171 additions and 0 deletions
140
apps/desktop/e2e/correction-session-switch.spec.ts
Normal file
140
apps/desktop/e2e/correction-session-switch.spec.ts
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
/**
|
||||
* Regression coverage for a correction sent during a live response, then a
|
||||
* warm session switch away and back. The correction is an accepted user turn,
|
||||
* not an optimistic duplicate of the original prompt, and its relative place
|
||||
* in the transcript must survive the resume reconciliation.
|
||||
*/
|
||||
|
||||
import { type TestInfo } from '@playwright/test'
|
||||
|
||||
import { expect, test, type Page } from './test'
|
||||
|
||||
import { type MockBackendFixture, setupMockBackend, waitForAppReady } from './fixtures'
|
||||
import { CORRECTION_SWITCH_TRIGGER, MOCK_REPLY } from './mock-server'
|
||||
|
||||
const OTHER_SESSION_PROMPT = 'E2E persisted session used for a warm resume.'
|
||||
const ORIGINAL_PROMPT = `${CORRECTION_SWITCH_TRIGGER}: original prompt must remain singular after a correction.`
|
||||
const CORRECTION = 'E2E correction must stay after the original prompt.'
|
||||
const TOOL_STARTED = 'Checking the long-running task before I continue.'
|
||||
const CORRECTED_REPLY = 'The corrected task finished.'
|
||||
|
||||
async function send(page: Page, text: string): Promise<void> {
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
await composer.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
await composer.click()
|
||||
await composer.type(text, { delay: 5 })
|
||||
await page.keyboard.press('Enter')
|
||||
}
|
||||
|
||||
async function waitForTranscriptText(page: Page, text: string): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
(expected: string) => (document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(expected),
|
||||
text,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
}
|
||||
|
||||
async function textNodeOccurrences(page: Page, text: string): Promise<number> {
|
||||
return page.evaluate((expected: string) => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return 0
|
||||
|
||||
const walker = document.createTreeWalker(viewport, NodeFilter.SHOW_TEXT)
|
||||
let count = 0
|
||||
while (walker.nextNode()) {
|
||||
if (walker.currentNode.textContent?.includes(expected)) {
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
return count
|
||||
}, text)
|
||||
}
|
||||
|
||||
async function transcriptTextOrder(page: Page): Promise<string[]> {
|
||||
return page.evaluate(() => {
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
if (!viewport) return []
|
||||
|
||||
return Array.from(viewport.querySelectorAll<HTMLElement>('[data-role="message"], [data-message-id]'))
|
||||
.map(message => message.textContent?.trim() ?? '')
|
||||
.filter(Boolean)
|
||||
})
|
||||
}
|
||||
|
||||
async function openFreshDraft(page: Page, priorSessionText: string): Promise<void> {
|
||||
await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click()
|
||||
await page.waitForFunction(
|
||||
(priorText: string) => !(document.querySelector('[data-slot="aui_thread-viewport"]')?.textContent ?? '').includes(priorText),
|
||||
priorSessionText,
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
}
|
||||
|
||||
async function openSidebarSession(page: Page, sidebarText: string, expectedTranscriptText: string): Promise<void> {
|
||||
const row = page.locator('[data-slot="sidebar"] button').filter({ hasText: sidebarText }).first()
|
||||
await row.waitFor({ state: 'visible', timeout: 30_000 })
|
||||
await row.click()
|
||||
await waitForTranscriptText(page, expectedTranscriptText)
|
||||
}
|
||||
|
||||
async function reopenOriginalSession(page: Page): Promise<void> {
|
||||
// A still-running tool has not generated a final title yet, so the sidebar
|
||||
// retains the source prompt as its provisional session title.
|
||||
await openSidebarSession(page, ORIGINAL_PROMPT, ORIGINAL_PROMPT)
|
||||
}
|
||||
|
||||
function relevantOrder(messages: string[]): string[] {
|
||||
return messages.filter(message => message.includes(ORIGINAL_PROMPT) || message.includes(CORRECTION))
|
||||
}
|
||||
|
||||
test.describe('correction session switch', () => {
|
||||
let fixture: MockBackendFixture | null = null
|
||||
|
||||
test.beforeEach(async () => {
|
||||
fixture = await setupMockBackend()
|
||||
await waitForAppReady(fixture, 120_000)
|
||||
})
|
||||
|
||||
test.afterEach(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test('keeps a live correction in place and does not duplicate its original prompt after switching sessions', async ({}, testInfo: TestInfo) => {
|
||||
const { page } = fixture!
|
||||
|
||||
// A blank draft does not exercise session hydration. Seed a real second
|
||||
// session first, matching the observed switch between two saved chats.
|
||||
await send(page, OTHER_SESSION_PROMPT)
|
||||
await waitForTranscriptText(page, MOCK_REPLY)
|
||||
await openFreshDraft(page, OTHER_SESSION_PROMPT)
|
||||
|
||||
await send(page, ORIGINAL_PROMPT)
|
||||
await waitForTranscriptText(page, TOOL_STARTED)
|
||||
await waitForTranscriptText(page, ORIGINAL_PROMPT)
|
||||
|
||||
// The historical session redirected while a foreground terminal task was
|
||||
// running. Enter records the accepted correction at the next tool boundary.
|
||||
await send(page, CORRECTION)
|
||||
await waitForTranscriptText(page, CORRECTION)
|
||||
|
||||
const orderBeforeSwitch = relevantOrder(await transcriptTextOrder(page))
|
||||
expect(orderBeforeSwitch).toEqual([ORIGINAL_PROMPT, CORRECTION])
|
||||
expect(await textNodeOccurrences(page, ORIGINAL_PROMPT)).toBe(1)
|
||||
expect(await textNodeOccurrences(page, CORRECTION)).toBe(1)
|
||||
await page.screenshot({ path: testInfo.outputPath('correction-before-session-switch.png') })
|
||||
|
||||
// Reproduce the observed race: switch to another persisted session while
|
||||
// the foreground tool is live, then return before its redirect settles.
|
||||
await openSidebarSession(page, MOCK_REPLY, OTHER_SESSION_PROMPT)
|
||||
await reopenOriginalSession(page)
|
||||
await page.waitForTimeout(500)
|
||||
await page.screenshot({ path: testInfo.outputPath('correction-after-warm-resume.png') })
|
||||
|
||||
expect(relevantOrder(await transcriptTextOrder(page))).toEqual(orderBeforeSwitch)
|
||||
expect(await textNodeOccurrences(page, ORIGINAL_PROMPT)).toBe(1)
|
||||
expect(await textNodeOccurrences(page, CORRECTION)).toBe(1)
|
||||
|
||||
await waitForTranscriptText(page, CORRECTED_REPLY)
|
||||
})
|
||||
})
|
||||
|
|
@ -97,6 +97,9 @@ let _sidebarCrossIndex = 0
|
|||
/** Per-server counter for the queue-stop script. */
|
||||
let _queueStopIndex = 0
|
||||
|
||||
/** Per-server counter for the correction/session-switch script. */
|
||||
let _correctionSwitchIndex = 0
|
||||
|
||||
/** User messages received by the mock, for E2E assertions on real submits. */
|
||||
const _receivedUserTexts: string[] = []
|
||||
|
||||
|
|
@ -106,6 +109,7 @@ function resetScriptIndex(): void {
|
|||
_sidebarScriptIndex = 0
|
||||
_sidebarCrossIndex = 0
|
||||
_queueStopIndex = 0
|
||||
_correctionSwitchIndex = 0
|
||||
_receivedUserTexts.length = 0
|
||||
}
|
||||
|
||||
|
|
@ -193,6 +197,19 @@ const QUEUE_STOP_SCRIPT: ScriptedTurn[] = [
|
|||
{ text: 'The paused task completed.' },
|
||||
]
|
||||
|
||||
// The reported correction arrived while a foreground tool was still running.
|
||||
// Keep that boundary open long enough for the renderer to redirect the turn,
|
||||
// then let the next model request complete normally.
|
||||
const CORRECTION_SWITCH_SCRIPT: ScriptedTurn[] = [
|
||||
{
|
||||
text: 'Checking the long-running task before I continue.',
|
||||
toolCalls: [{ name: 'terminal', args: { command: 'sleep 5' } }],
|
||||
},
|
||||
{ text: 'The corrected task finished.' },
|
||||
]
|
||||
|
||||
export const CORRECTION_SWITCH_TRIGGER = 'E2E_CORRECTION_SWITCH_TRIGGER'
|
||||
|
||||
/**
|
||||
* A marker that makes the mock emit a real blocking clarify tool call. Tests
|
||||
* use it to hold a turn open while exercising busy-composer interactions.
|
||||
|
|
@ -313,6 +330,9 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
|
|||
const isSidebarTrigger = userText.includes('E2E_SIDEBAR_TRIGGER')
|
||||
const isSidebarCrossTrigger = userText.includes('E2E_SIDEBAR_CROSS')
|
||||
const isQueueStopTrigger = userText.includes('E2E_QUEUE_STOP_TRIGGER')
|
||||
const isCorrectionSwitchTrigger = messages.some(
|
||||
message => typeof message?.content === 'string' && message.content.includes(CORRECTION_SWITCH_TRIGGER),
|
||||
)
|
||||
|
||||
if (includesBlockingClarifyTrigger(parsed.messages)) {
|
||||
if (stream) {
|
||||
|
|
@ -334,6 +354,17 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
|
|||
return
|
||||
}
|
||||
|
||||
if (isCorrectionSwitchTrigger) {
|
||||
const turn = CORRECTION_SWITCH_SCRIPT[_correctionSwitchIndex] ?? CORRECTION_SWITCH_SCRIPT[CORRECTION_SWITCH_SCRIPT.length - 1]
|
||||
_correctionSwitchIndex++
|
||||
if (stream) {
|
||||
streamScriptedTurn(res, model, turn)
|
||||
} else {
|
||||
nonStreamingScriptedTurn(res, model, turn)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (isSidebarCrossTrigger) {
|
||||
const turn = SIDEBAR_CROSS_SCRIPT[_sidebarCrossIndex] ?? SIDEBAR_CROSS_SCRIPT[SIDEBAR_CROSS_SCRIPT.length - 1]
|
||||
_sidebarCrossIndex++
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue