diff --git a/apps/desktop/scripts/perf/baseline.json b/apps/desktop/scripts/perf/baseline.json index e24e85fb9d59..392993f618c6 100644 --- a/apps/desktop/scripts/perf/baseline.json +++ b/apps/desktop/scripts/perf/baseline.json @@ -1,9 +1,9 @@ { "_meta": { - "note": "Median of 5 runs, darwin-arm64, `--spawn --prod` (PRODUCTION minified renderer, real boot — no fake-boot). Representative shipped numbers, not dev-inflated. cold-start uses a fresh unique-port spawn per run and its marks are process-spawn wall clock (spawn_to_*) or renderer nav-relative (dom_*). Re-baseline per device with `--update-baseline`; tolerances are loose for cross-machine/disk variance.", + "note": "Median of 5 runs, darwin-arm64, `--spawn --prod` (PRODUCTION minified renderer, real boot — no fake-boot). Representative shipped numbers, not dev-inflated. cold-start reuses one profile so the V8 code cache is WARM (what users get after first launch, ~1.0s); a fresh-profile first launch is ~+400ms (measure with `--cold-fresh`). Marks are process-spawn wall clock (spawn_to_*) or renderer nav-relative (dom_*). Re-baseline per device with `--update-baseline`; tolerances loose for cross-machine/disk variance.", "platform": "darwin-arm64", "node": "v24.11.0", - "updated": "2026-07-19T22:59:21.605Z" + "updated": "2026-07-19T23:16:01.227Z" }, "scenarios": { "stream": { @@ -49,11 +49,11 @@ "tolAbs": 150 }, "metrics": { - "spawn_to_cdp_ms": 1098, - "spawn_to_driver_ms": 1482, - "dom_interactive_ms": 794, - "dom_content_loaded_ms": 1057, - "nav_to_read_ms": 1209 + "spawn_to_cdp_ms": 606, + "spawn_to_driver_ms": 984, + "dom_interactive_ms": 324, + "dom_content_loaded_ms": 574, + "nav_to_read_ms": 721 } } } diff --git a/apps/desktop/scripts/perf/lib/launch.mjs b/apps/desktop/scripts/perf/lib/launch.mjs index 46a9cba7e1e9..08410bbf2869 100644 --- a/apps/desktop/scripts/perf/lib/launch.mjs +++ b/apps/desktop/scripts/perf/lib/launch.mjs @@ -326,6 +326,65 @@ export async function startIsolatedInstance({ } } +// Representative cold-start sampling. A fresh --user-data-dir means a COLD V8 +// code cache and worst-case bundle recompile every run (~+400ms measured); real +// users reuse their profile, so a warm cache is the representative case. We reuse +// ONE profile across runs: run 0 warms the cache (discarded), runs 1..N are the +// warm samples. Each run steps the port so a just-killed instance can't be +// re-attached, and we pause between runs so the single-instance lock releases. +export async function coldStartSamples({ runs = 3, port = 9222, devPort = 5174, prod = false, warm = true } = {}) { + const pickNumeric = timings => Object.fromEntries(Object.entries(timings).filter(([, v]) => typeof v === 'number')) + const samples = [] + + if (warm) { + // Shared profile across runs: run 0 warms the V8 code cache (discarded), + // runs 1..N are the representative warm samples. + const home = mkdtempSync(join(tmpdir(), 'hermes-perf-cold-home-')) + const userDataDir = mkdtempSync(join(tmpdir(), 'hermes-perf-cold-ud-')) + seedConfigFrom(join(homedir(), '.hermes'), home) + + try { + for (let i = 0; i <= runs; i++) { + const inst = await startIsolatedInstance({ + port: port + i, + devPort: devPort + i, + prod, + coldStart: true, + hermesHome: home, + userDataDir, + seedConfig: false + }) + + if (i > 0) { + samples.push(pickNumeric(inst.timings)) + } + + inst.teardown() + await sleep(2500) // let the single-instance lock release before reuse + } + } finally { + for (const dir of [home, userDataDir]) { + try { + rmSync(dir, { recursive: true, force: true }) + } catch { + // best-effort + } + } + } + } else { + // Worst case: a fresh profile per run → cold code cache every launch + // (first-launch-after-install). startIsolatedInstance makes+removes its dirs. + for (let i = 0; i < runs; i++) { + const inst = await startIsolatedInstance({ port: port + i, devPort: devPort + i, prod, coldStart: true }) + samples.push(pickNumeric(inst.timings)) + inst.teardown() + await sleep(2500) + } + } + + return samples +} + // Read First Contentful Paint + time-to-composer from the renderer, relative to // its navigation start (the process-spawn deltas live in `timings`). async function readBootMarks(cdp) { diff --git a/apps/desktop/scripts/perf/run.mjs b/apps/desktop/scripts/perf/run.mjs index 16db25f05090..6d4dc757c249 100644 --- a/apps/desktop/scripts/perf/run.mjs +++ b/apps/desktop/scripts/perf/run.mjs @@ -29,7 +29,7 @@ import { fileURLToPath } from 'node:url' import { withCpuProfile } from './lib/cdp.mjs' import { compareScenario, loadBaseline, updateBaseline } from './lib/baseline.mjs' -import { attach, buildProdRenderer, startIsolatedInstance } from './lib/launch.mjs' +import { attach, buildProdRenderer, coldStartSamples, startIsolatedInstance } from './lib/launch.mjs' import { cpuProfileTopSelf, median } from './lib/stats.mjs' import { CI_SCENARIOS, SCENARIOS } from './scenarios/index.mjs' @@ -149,20 +149,11 @@ async function main() { process.exit(2) } - const perRun = [] + // Representative WARM-cache samples (see coldStartSamples). Pass --cold-fresh + // to instead measure the worst-case first-launch (cold code cache). + const perRun = await coldStartSamples({ runs, port, devPort, prod, warm: !('cold-fresh' in flags) }) - for (let i = 0; i < runs; i++) { - // Unique debug + dev port per run: a just-killed instance can keep :9222 - // held for a beat, and reusing it makes the next CDP.connect attach to the - // dying instance (garbage timings). Stepping the port sidesteps the race. - const inst = await startIsolatedInstance({ port: port + i, devPort: devPort + i, prod, coldStart: true }) - // Forward every numeric boot mark; only the baseline keys are gated, the - // rest (dom_interactive, main_script_kb, …) are reported for composition. - perRun.push(Object.fromEntries(Object.entries(inst.timings).filter(([, v]) => typeof v === 'number'))) - inst.teardown() - } - - record('cold-start', 'cold', medianMetrics(perRun), { runs }) + record('cold-start', 'cold', medianMetrics(perRun), { runs, warm: !('cold-fresh' in flags) }) } // Steady-state scenarios share one persistent connection.