mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
test(desktop): e2e proving session-load re-render bug
Adds an E2E test that reproduces the multi-pass transcript re-render bug when loading a large previous session. The desktop's resumeSession path does up to three message-set passes (warm cache → session.activate projection → REST persisted transcript), each replacing the $messages atom and rebuilding the transcript DOM — visible as a flicker on large sessions. The test: - Seeds a real 53-message session (exported from state.db, stripped to minimal stubs — 13.5KB) into a sandbox HERMES_HOME via SessionDB.import_sessions before launch - Opens the session from the sidebar - Instruments the thread viewport with a MutationObserver to count DOM mutation bursts (groups separated by a 30ms gap) - Asserts >= 2 bursts, proving the transcript was rebuilt more than once Also exports createSandbox, writeMockProviderConfig, writeEnvFile, buildAppEnv, findElectron, and launchDesktop from fixtures.ts so the new spec can compose its own seeded-backend fixture without duplicating the sandbox/config/launch logic.
This commit is contained in:
parent
71e3b7d832
commit
8d2883abf8
4 changed files with 400 additions and 6 deletions
|
|
@ -96,7 +96,7 @@ export interface Sandbox {
|
|||
cleanup: () => void
|
||||
}
|
||||
|
||||
function createSandbox(prefix: string): Sandbox {
|
||||
export function createSandbox(prefix: string): Sandbox {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), `hermes-e2e-${prefix}-${Math.random()}`))
|
||||
const hermesHome = path.join(root, 'hermes-home')
|
||||
const userDataDir = path.join(root, 'electron-user-data')
|
||||
|
|
@ -140,7 +140,7 @@ function createSandbox(prefix: string): Sandbox {
|
|||
* mock inference server. The provider is set as the active model provider so
|
||||
* the desktop app skips onboarding and boots straight to the chat UI.
|
||||
*/
|
||||
function writeMockProviderConfig(hermesHome: string, mockUrl: string): void {
|
||||
export function writeMockProviderConfig(hermesHome: string, mockUrl: string): void {
|
||||
const configPath = path.join(hermesHome, 'config.yaml')
|
||||
|
||||
const config = `# Auto-generated by E2E test fixtures
|
||||
|
|
@ -165,7 +165,7 @@ providers:
|
|||
* Write a minimal .env with the mock API key. The key_env in config.yaml
|
||||
* references MOCK_API_KEY, so the backend resolves credentials from here.
|
||||
*/
|
||||
function writeEnvFile(hermesHome: string, apiKey = 'e2e-mock-key'): void {
|
||||
export function writeEnvFile(hermesHome: string, apiKey = 'e2e-mock-key'): void {
|
||||
const envPath = path.join(hermesHome, '.env')
|
||||
fs.writeFileSync(envPath, `MOCK_API_KEY=${apiKey}\n`, 'utf8')
|
||||
}
|
||||
|
|
@ -193,7 +193,7 @@ function writeEmptyConfig(hermesHome: string): void {
|
|||
* - HERMES_DESKTOP_APP_NAME → unique-ish per test (avoids single-instance lock)
|
||||
* - XDG_RUNTIME_DIR → ensure Electron has a writable runtime dir on Linux
|
||||
*/
|
||||
function buildAppEnv(sandbox: Sandbox, extra: Record<string, string> = {}): Record<string, string> {
|
||||
export function buildAppEnv(sandbox: Sandbox, extra: Record<string, string> = {}): Record<string, string> {
|
||||
const clean = stripCredentials(process.env)
|
||||
|
||||
// XDG_RUNTIME_DIR is needed for Electron on Linux when running in a
|
||||
|
|
@ -252,7 +252,7 @@ function assertDistBuilt(): void {
|
|||
* Find the Electron binary. In the nix devshell, `electron` is on PATH.
|
||||
* As a fallback, use the node_modules/.bin/electron from the desktop package.
|
||||
*/
|
||||
function findElectron(): string {
|
||||
export function findElectron(): string {
|
||||
// In dev mode, we use the `electron` binary directly (not the packaged app).
|
||||
// The dev:electron script in package.json does exactly this: `electron .`
|
||||
// after building. We replicate that here.
|
||||
|
|
@ -283,7 +283,7 @@ function findElectron(): string {
|
|||
* @param env - the process environment (already has HERMES_HOME etc.)
|
||||
* @returns the ElectronApplication + first Page
|
||||
*/
|
||||
async function launchDesktop(
|
||||
export async function launchDesktop(
|
||||
env: Record<string, string>,
|
||||
): Promise<{ app: ElectronApplication; page: Page }> {
|
||||
assertDistBuilt()
|
||||
|
|
|
|||
1
apps/desktop/e2e/fixtures/large-session.json
Normal file
1
apps/desktop/e2e/fixtures/large-session.json
Normal file
File diff suppressed because one or more lines are too long
337
apps/desktop/e2e/large-session-reload.spec.ts
Normal file
337
apps/desktop/e2e/large-session-reload.spec.ts
Normal file
|
|
@ -0,0 +1,337 @@
|
|||
/**
|
||||
* E2E test: loading a large previous session re-renders the transcript
|
||||
* multiple times.
|
||||
*
|
||||
* The desktop's `resumeSession` path does up to three message-set passes
|
||||
* when loading a stored session:
|
||||
*
|
||||
* 1. Warm cache (if the runtime id is still live) → immediate paint
|
||||
* 2. `session.activate` projection → reconcileAuthoritativeMessages
|
||||
* 3. REST `getSessionMessages` persisted transcript → reconcile again
|
||||
*
|
||||
* Each pass calls `syncSessionStateToView` which replaces the `$messages`
|
||||
* atom, causing the assistant-ui thread to re-render the whole transcript.
|
||||
* On a large session (53 messages here) this is visible as a flicker —
|
||||
* the transcript appears, then gets replaced, then settles.
|
||||
*
|
||||
* This test seeds a real exported session (53 messages — 1 user, 11
|
||||
* assistant, 41 tool) into the sandbox's state.db BEFORE launching the app,
|
||||
* so the session appears in the sidebar on first load. It then clicks the
|
||||
* session row and instruments DOM mutations on the thread viewport to prove
|
||||
* the transcript DOM was rebuilt more than once during the load.
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import * as path from 'node:path'
|
||||
|
||||
import { _electron, expect, type ElectronApplication, type Page, test } from '@playwright/test'
|
||||
|
||||
import {
|
||||
buildAppEnv,
|
||||
createSandbox,
|
||||
findElectron,
|
||||
launchDesktop,
|
||||
writeEnvFile,
|
||||
writeMockProviderConfig,
|
||||
waitForAppReady,
|
||||
type Sandbox,
|
||||
} from './fixtures'
|
||||
import { startMockServer } from './mock-server'
|
||||
|
||||
// ─── Constants ─────────────────────────────────────────────────────────
|
||||
|
||||
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
|
||||
const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..')
|
||||
const FIXTURE_PATH = path.resolve(import.meta.dirname, 'fixtures', 'large-session.json')
|
||||
const SEED_SCRIPT = path.resolve(import.meta.dirname, 'scripts', 'seed_session_db.py')
|
||||
|
||||
const SESSION_TITLE = 'Auditing and Removing Brew Pip Support'
|
||||
|
||||
// ─── Fixture data ──────────────────────────────────────────────────────
|
||||
|
||||
/** The exported session from a real Hermes state.db (53 messages). */
|
||||
function loadSessionFixture(): Record<string, unknown> {
|
||||
const raw = readFileSync(FIXTURE_PATH, 'utf8')
|
||||
return JSON.parse(raw) as Record<string, unknown>
|
||||
}
|
||||
|
||||
// ─── Seeded mock backend fixture ───────────────────────────────────────
|
||||
|
||||
interface SeededMockBackendFixture {
|
||||
app: ElectronApplication
|
||||
page: Page
|
||||
mockUrl: string
|
||||
sandbox: Sandbox
|
||||
cleanup: () => Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up a mock-backend environment with a pre-seeded session in state.db.
|
||||
*
|
||||
* Like setupMockBackend but seeds the large-session.json fixture into the
|
||||
* sandbox's state.db BEFORE launching the app, so the session appears in
|
||||
* the sidebar on first load.
|
||||
*/
|
||||
async function setupSeededMockBackend(): Promise<SeededMockBackendFixture> {
|
||||
// 1. Start mock inference server
|
||||
const mock = await startMockServer()
|
||||
|
||||
// 2. Create sandbox + write config
|
||||
const sandbox = createSandbox('seeded')
|
||||
writeMockProviderConfig(sandbox.hermesHome, mock.url)
|
||||
writeEnvFile(sandbox.hermesHome)
|
||||
|
||||
// 3. Seed the state.db with the large session fixture BEFORE the app
|
||||
// starts. The backend creates state.db on first run, but if it already
|
||||
// exists with the right schema, the backend reuses it.
|
||||
const stateDbPath = path.join(sandbox.hermesHome, 'state.db')
|
||||
const pythonBin = spawnSync('which', ['python3'], { encoding: 'utf8' })
|
||||
const python = pythonBin.status === 0 ? pythonBin.stdout.trim() : 'python3'
|
||||
|
||||
const seedResult = spawnSync(
|
||||
python,
|
||||
[SEED_SCRIPT, stateDbPath, FIXTURE_PATH],
|
||||
{
|
||||
encoding: 'utf8',
|
||||
cwd: REPO_ROOT,
|
||||
env: { ...process.env, PYTHONPATH: REPO_ROOT },
|
||||
},
|
||||
)
|
||||
|
||||
if (seedResult.status !== 0) {
|
||||
throw new Error(
|
||||
`Failed to seed state.db:\nstdout: ${seedResult.stdout}\nstderr: ${seedResult.stderr}`,
|
||||
)
|
||||
}
|
||||
|
||||
// 4. Build env + launch
|
||||
const env = buildAppEnv(sandbox)
|
||||
const { app, page } = await launchDesktop(env)
|
||||
|
||||
return {
|
||||
app,
|
||||
page,
|
||||
mockUrl: mock.url,
|
||||
sandbox,
|
||||
cleanup: async () => {
|
||||
await app.close().catch(() => undefined)
|
||||
await mock.close()
|
||||
sandbox.cleanup()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Test ───────────────────────────────────────────────────────────────
|
||||
|
||||
let fixture: SeededMockBackendFixture | null = null
|
||||
|
||||
test.beforeAll(async () => {
|
||||
fixture = await setupSeededMockBackend()
|
||||
await waitForAppReady(fixture!, 120_000)
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test.describe('loading a large previous session', () => {
|
||||
test('transcript re-renders multiple times during load', async () => {
|
||||
const page = fixture!.page
|
||||
|
||||
// ── 1. Wait for the session to appear in the sidebar. The session was
|
||||
// seeded into state.db before launch, so it appears once the session
|
||||
// list loads (shortly after the gateway opens). We don't need to wait
|
||||
// for the "Waking up" profile placeholder to disappear — that's the
|
||||
// empty-chat area, not the sidebar.
|
||||
const sessionRow = page
|
||||
.locator('[data-slot="sidebar"] button')
|
||||
.filter({ hasText: SESSION_TITLE })
|
||||
.first()
|
||||
|
||||
await sessionRow.waitFor({ state: 'visible', timeout: 60_000 })
|
||||
|
||||
// ── 2. Instrument DOM rebuilds in the thread viewport during load.
|
||||
// Each reconciliation pass in resumeSession replaces the $messages
|
||||
// atom (warm cache → session.activate projection → REST persisted
|
||||
// transcript), causing assistant-ui to tear down and rebuild the
|
||||
// message DOM nodes. A MutationObserver catches these synchronous
|
||||
// DOM mutations; we coalesce them into bursts (groups separated by
|
||||
// a 30ms gap) — each burst is a distinct re-render cycle.
|
||||
//
|
||||
// A single clean load would produce one burst. The bug produces
|
||||
// 2+ because the three reconciliation passes each rebuild the DOM.
|
||||
await page.evaluate(() => {
|
||||
const w = window as unknown as {
|
||||
__RENDER_COUNT__?: {
|
||||
bursts: number
|
||||
mutations: number
|
||||
timeline: { burst: number; time: number; mutations: number }[]
|
||||
stopped: boolean
|
||||
}
|
||||
}
|
||||
|
||||
const state = {
|
||||
bursts: 0,
|
||||
mutations: 0,
|
||||
timeline: [] as { burst: number; time: number; mutations: number }[],
|
||||
stopped: false,
|
||||
}
|
||||
w.__RENDER_COUNT__ = state
|
||||
|
||||
const viewport = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
|
||||
if (!viewport) {
|
||||
return
|
||||
}
|
||||
|
||||
let currentBatch = 0
|
||||
let flushTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const flush = () => {
|
||||
flushTimer = null
|
||||
if (currentBatch > 0 && !state.stopped) {
|
||||
state.bursts += 1
|
||||
state.timeline.push({ burst: state.bursts, time: Date.now(), mutations: currentBatch })
|
||||
currentBatch = 0
|
||||
}
|
||||
}
|
||||
|
||||
const observer = new MutationObserver(records => {
|
||||
if (state.stopped) {
|
||||
return
|
||||
}
|
||||
|
||||
let batchAdded = 0
|
||||
for (const record of records) {
|
||||
if (record.type === 'childList') {
|
||||
state.mutations += 1
|
||||
batchAdded += 1
|
||||
}
|
||||
}
|
||||
|
||||
if (batchAdded > 0) {
|
||||
currentBatch += batchAdded
|
||||
if (flushTimer) {
|
||||
clearTimeout(flushTimer)
|
||||
}
|
||||
flushTimer = setTimeout(flush, 30)
|
||||
}
|
||||
})
|
||||
|
||||
observer.observe(viewport, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
attributes: false,
|
||||
characterData: false,
|
||||
})
|
||||
})
|
||||
|
||||
// ── 3. Click the session row to trigger resumeSession(). This is the
|
||||
// action that triggers the multi-pass message loading.
|
||||
await sessionRow.click()
|
||||
|
||||
// ── 4. Wait for the transcript to settle — message text should be
|
||||
// visible in the thread viewport.
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const vp = document.querySelector('[data-slot="aui_thread-viewport"]')
|
||||
|
||||
if (!vp) {
|
||||
return false
|
||||
}
|
||||
|
||||
const text = vp.textContent ?? ''
|
||||
|
||||
// The session is about auditing brew/pip support — look for
|
||||
// a distinctive phrase from the conversation.
|
||||
return text.includes('audit') || text.includes('brew') || text.includes('packaging')
|
||||
},
|
||||
undefined,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
|
||||
// ── 5. Take a screenshot of the loaded transcript.
|
||||
await page.screenshot({
|
||||
path: path.resolve(import.meta.dirname, '..', 'test-results', 'large-session-loaded.png'),
|
||||
fullPage: false,
|
||||
})
|
||||
|
||||
// ── 6. Give the reconciliation passes time to complete. The three
|
||||
// passes (warm cache → session.activate → REST persisted) each
|
||||
// fire an async reconcile. Wait for the mutation bursts to settle
|
||||
// — no new burst for 1 second means the load is done.
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const w = window as unknown as {
|
||||
__RENDER_COUNT__?: {
|
||||
bursts: number
|
||||
timeline: { burst: number; time: number; mutations: number }[]
|
||||
}
|
||||
}
|
||||
|
||||
const state = w.__RENDER_COUNT__
|
||||
|
||||
if (!state) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (state.bursts === 0) {
|
||||
return false
|
||||
}
|
||||
|
||||
const last = state.timeline[state.timeline.length - 1]
|
||||
|
||||
return Date.now() - last.time > 1000
|
||||
},
|
||||
undefined,
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
|
||||
// ── 7. Stop counting and read the results.
|
||||
const results = await page.evaluate(() => {
|
||||
const w = window as unknown as {
|
||||
__RENDER_COUNT__?: {
|
||||
bursts: number
|
||||
mutations: number
|
||||
timeline: { burst: number; time: number; mutations: number }[]
|
||||
stopped: boolean
|
||||
}
|
||||
}
|
||||
|
||||
const state = w.__RENDER_COUNT__
|
||||
|
||||
if (state) {
|
||||
state.stopped = true
|
||||
}
|
||||
|
||||
return state
|
||||
? {
|
||||
bursts: state.bursts,
|
||||
mutations: state.mutations,
|
||||
timeline: state.timeline.map(t => ({ burst: t.burst, mutations: t.mutations })),
|
||||
}
|
||||
: null
|
||||
})
|
||||
|
||||
// ── 8. Assert the bug: the transcript re-rendered more than once.
|
||||
//
|
||||
// A single clean load would produce exactly one mutation burst —
|
||||
// the initial DOM build. The bug produces 2+ bursts because the
|
||||
// three reconciliation passes (warm cache, session.activate, REST
|
||||
// persisted) each replace the $messages atom and rebuild the
|
||||
// transcript DOM.
|
||||
expect(results, 'render count data should have been collected').not.toBeNull()
|
||||
|
||||
if (results) {
|
||||
expect(
|
||||
results.bursts,
|
||||
`expected >= 2 mutation bursts (multiple re-renders during load), ` +
|
||||
`got ${results.bursts}: ${JSON.stringify(results.timeline)}`,
|
||||
).toBeGreaterThanOrEqual(2)
|
||||
}
|
||||
})
|
||||
})
|
||||
56
apps/desktop/e2e/scripts/seed_session_db.py
Normal file
56
apps/desktop/e2e/scripts/seed_session_db.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Seed a Hermes state.db with a session exported from a real conversation.
|
||||
|
||||
Usage: seed_session_db.py <state_db_path> <fixture_json_path>
|
||||
|
||||
Creates the database with the full SessionDB schema (if it doesn't exist)
|
||||
and imports the session from the JSON fixture. Uses the real
|
||||
SessionDB.import_sessions() so the data shape matches what the desktop
|
||||
backend expects.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add the repo root to sys.path so we can import hermes_state.
|
||||
# The script is invoked from apps/desktop/e2e/ — repo root is ../../..
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
sys.path.insert(0, str(repo_root))
|
||||
|
||||
from hermes_state import SessionDB # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
print(f"Usage: {sys.argv[0]} <state_db_path> <fixture_json_path>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
db_path = Path(sys.argv[1])
|
||||
fixture_path = Path(sys.argv[2])
|
||||
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(fixture_path, "r", encoding="utf-8") as f:
|
||||
session_data = json.load(f)
|
||||
|
||||
db = SessionDB(db_path=db_path)
|
||||
result = db.import_sessions([session_data])
|
||||
|
||||
if not result.get("ok"):
|
||||
print(f"Import failed: {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
imported = result.get("imported", 0)
|
||||
skipped = result.get("skipped", 0)
|
||||
errors = result.get("errors", [])
|
||||
|
||||
if errors:
|
||||
print(f"Import had errors: {errors}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Seeded {imported} session(s), skipped {skipped} → {db_path}")
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue