diff --git a/agent/background_review.py b/agent/background_review.py index fa4de508e19..564c5441996 100644 --- a/agent/background_review.py +++ b/agent/background_review.py @@ -27,6 +27,131 @@ from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Background-review aux-model selector + routed digest. +# +# The review fork runs on the MAIN model by default ("auto"), replaying the +# full conversation — already warm in the prompt cache, so cheap cache reads. +# Optimal and unchanged. A user can route the review to a different, cheaper +# model via auxiliary.background_review.{provider,model}. A different model +# cannot reuse the parent's cache (different key), so the fork is cold +# regardless — replaying the full transcript would just cold-write it. So when +# (and only when) routed to a different model, we replay a compact DIGEST to +# minimise cold-written tokens. Same model -> full replay; different model -> +# digest. That's the whole policy. +# --------------------------------------------------------------------------- + + +def _resolve_review_runtime(agent: Any) -> Dict[str, Any]: + """Resolve provider/model/credentials for the review fork. + + Default (auto / unset / same as parent): inherit the parent's live runtime + (with codex_app_server -> codex_responses downgrade). ``routed`` is False — + the fork uses the main model and the warm cache, exactly as before. When + ``auxiliary.background_review.{provider,model}`` names a concrete model + different from the parent's, resolve that runtime and set ``routed=True``. + """ + parent_runtime = agent._current_main_runtime() + parent_api_mode = parent_runtime.get("api_mode") or None + if parent_api_mode == "codex_app_server": + parent_api_mode = "codex_responses" + parent = { + "provider": agent.provider, + "model": agent.model, + "api_key": parent_runtime.get("api_key") or None, + "base_url": parent_runtime.get("base_url") or None, + "api_mode": parent_api_mode, + "routed": False, + } + try: + from hermes_cli.config import load_config + cfg = load_config() + except Exception: + return parent + aux = cfg.get("auxiliary", {}) if isinstance(cfg.get("auxiliary"), dict) else {} + task = aux.get("background_review", {}) if isinstance(aux.get("background_review"), dict) else {} + task_provider = (str(task.get("provider", "")).strip() or None) + task_model = (str(task.get("model", "")).strip() or None) + task_base_url = (str(task.get("base_url", "")).strip() or None) + task_api_key = (str(task.get("api_key", "")).strip() or None) + if not (task_provider and task_provider != "auto" and task_model): + return parent + if task_provider == (agent.provider or "") and task_model == (agent.model or ""): + return parent # same model/provider as parent -> not routed + try: + from hermes_cli.runtime_provider import resolve_runtime_provider + rp = resolve_runtime_provider( + requested=task_provider, + target_model=task_model, + explicit_api_key=task_api_key, + explicit_base_url=task_base_url, + ) + return { + "provider": rp.get("provider") or task_provider, + "model": task_model, + "api_key": rp.get("api_key"), + "base_url": rp.get("base_url"), + "api_mode": rp.get("api_mode"), + "routed": True, + } + except Exception as e: + logger.debug("background-review aux routing failed (%s); using main model", e) + return parent + + +def _msg_text(m: Dict) -> str: + c = m.get("content") + if isinstance(c, str): + return c.strip() + if isinstance(c, list): + return " ".join(b.get("text", "") for b in c if isinstance(b, dict)).strip() + return "" + + +def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict]: + """Compact replay for the routed (different-model) path only. + + Keeps the recent ``tail`` messages verbatim, collapses older turns into one + synthetic user-role digest, preserving role alternation. Used ONLY when + routed to a different model (cache cold regardless, so fewer cold-written + tokens is a pure win). Never on the main-model path (full replay stays warm). + """ + msgs = list(messages_snapshot or []) + if len(msgs) <= tail: + return msgs + keep = msgs[-tail:] + while keep and isinstance(keep[0], dict) and keep[0].get("role") == "tool": + tail += 1 + if len(msgs) <= tail: + return msgs + keep = msgs[-tail:] + old = msgs[:-len(keep)] + lines: List[str] = [] + for m in old: + if not isinstance(m, dict): + continue + role = m.get("role") + text = _msg_text(m).replace("\n", " ") + if role == "user" and text: + lines.append(f"USER: {text[:300]}") + elif role == "assistant": + tcs = m.get("tool_calls") or [] + if tcs: + names = [(tc.get("function") or {}).get("name", "?") for tc in tcs if isinstance(tc, dict)] + lines.append(f"ASSISTANT[tools: {', '.join(names)}]") + if text: + lines.append(f"ASSISTANT: {text[:200]}") + digest = { + "role": "user", + "content": ( + "[Earlier conversation digest — older turns summarised to bound the " + "review's cold-write cost on the routed aux model. Recent turns " + "follow verbatim below.]\n" + "\n".join(lines) + ), + } + return [digest] + keep + + # Review-prompt strings — used by ``spawn_background_review_thread`` to build # the user-message that the forked review agent receives. AIAgent exposes # them as class attributes (``_MEMORY_REVIEW_PROMPT`` etc.) for back-compat; @@ -488,18 +613,13 @@ def _run_review_in_thread( # creds, or credential-pool setups where the resolver can't # reconstruct auth from scratch -- producing the spurious # "No LLM provider configured" warning at end of turn. - _parent_runtime = agent._current_main_runtime() - _parent_api_mode = _parent_runtime.get("api_mode") or None - # The review fork needs to call agent-loop tools (memory, - # skill_manage). Those tools require Hermes' own dispatch, - # which the codex_app_server runtime bypasses entirely - # (it runs the turn inside codex's subprocess). So when - # the parent is on codex_app_server, downgrade the review - # fork to codex_responses — same auth/credentials, but - # talks to the OpenAI Responses API directly so Hermes - # owns the loop and the agent-loop tools dispatch. - if _parent_api_mode == "codex_app_server": - _parent_api_mode = "codex_responses" + # _resolve_review_runtime() returns the parent's live runtime by + # default (routed=False; main model, warm cache), or — when the user + # set auxiliary.background_review.{provider,model} to a different + # model — that model's runtime (routed=True). The codex_app_server + # -> codex_responses downgrade is applied inside the resolver. + _rt = _resolve_review_runtime(agent) + _routed = bool(_rt.get("routed")) # skip_memory=True keeps the review fork from # touching external memory plugins (honcho, mem0, # supermemory, etc.). Without it, the fork's @@ -519,14 +639,14 @@ def _run_review_in_thread( # in the request body — Anthropic's cache key includes it. # (The runtime whitelist below still restricts dispatch.) review_agent = AIAgent( - model=agent.model, + model=_rt.get("model") or agent.model, max_iterations=16, quiet_mode=True, platform=agent.platform, - provider=agent.provider, - api_mode=_parent_api_mode, - base_url=_parent_runtime.get("base_url") or None, - api_key=_parent_runtime.get("api_key") or None, + provider=_rt.get("provider") or agent.provider, + api_mode=_rt.get("api_mode"), + base_url=_rt.get("base_url") or None, + api_key=_rt.get("api_key") or None, credential_pool=getattr(agent, "_credential_pool", None), parent_session_id=agent.session_id, enabled_toolsets=getattr(agent, "enabled_toolsets", None), @@ -565,15 +685,20 @@ def _run_review_in_thread( # issue #25322 and PR #17276 for the full analysis + # measured impact (~26% end-to-end cost reduction on # Sonnet 4.5). - review_agent._cached_system_prompt = agent._cached_system_prompt - # Defensive: pin session_start + session_id to the - # parent's so any code path that re-renders parts of - # the system prompt (compression, plugin hooks) still - # produces byte-identical output. The cached-prompt - # assignment above already short-circuits the normal - # rebuild path, but these pins guarantee parity even - # if a future code path bypasses the cache. - review_agent.session_start = agent.session_start + # Share the parent's warm cached system prompt ONLY when the review + # runs on the SAME model (not routed). When routed to a different + # model the parent's cached prompt is for the wrong model/cache key + # and would miss anyway, so let the routed fork build its own. + if not _routed: + review_agent._cached_system_prompt = agent._cached_system_prompt + # Defensive: pin session_start + session_id to the + # parent's so any code path that re-renders parts of + # the system prompt (compression, plugin hooks) still + # produces byte-identical output. The cached-prompt + # assignment above already short-circuits the normal + # rebuild path, but these pins guarantee parity even + # if a future code path bypasses the cache. + review_agent.session_start = agent.session_start review_agent.session_id = agent.session_id # The fork shares the parent's live session_id (pinned above for # prefix-cache parity). It is single-lifecycle and calls close() @@ -615,6 +740,13 @@ def _run_review_in_thread( ), ) try: + # Routed to a different model -> replay a digest (cache is cold + # on that model anyway, so minimise cold-written tokens). Same + # model -> replay the full snapshot (warm cache reads). + _review_history = ( + _digest_history(messages_snapshot) if _routed + else messages_snapshot + ) review_agent.run_conversation( user_message=( prompt @@ -622,7 +754,7 @@ def _run_review_in_thread( "management tools. Other tools will be denied " "at runtime — do not attempt them." ), - conversation_history=messages_snapshot, + conversation_history=_review_history, ) finally: clear_thread_tool_whitelist() diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index 94fff283893..ba67f036954 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -805,10 +805,11 @@ def try_shrink_image_parts_in_messages( Pillow couldn't help (caller should surface the original error). Strategy: look for ``image_url`` / ``input_image`` parts carrying a - ``data:image/...;base64,...`` payload. For each one whose encoded - size exceeds 4 MB (a safe target that slides under Anthropic's 5 MB - ceiling with header overhead) or whose longest side exceeds - ``max_dimension``, write the base64 to a tempfile, call + ``data:image/...;base64,...`` payload, plus Anthropic-native + ``{"type": "image", "source": {"type": "base64", ...}}`` blocks. + For each one whose encoded size exceeds 4 MB (a safe target that slides + under Anthropic's 5 MB ceiling with header overhead) or whose longest side + exceeds ``max_dimension``, write the base64 to a tempfile, call ``vision_tools._resize_image_for_vision`` to produce a smaller data URL, and substitute it in place. @@ -964,6 +965,28 @@ def try_shrink_image_parts_in_messages( logger.warning("image-shrink recovery: re-encode failed — %s", exc) return None, triggered_by is not None + def _source_to_data_url(source: Any) -> Optional[str]: + if not isinstance(source, dict) or source.get("type") != "base64": + return None + data = source.get("data") + if not isinstance(data, str) or not data: + return None + media_type = str(source.get("media_type") or "image/jpeg").strip() + if not media_type.startswith("image/"): + media_type = "image/jpeg" + return f"data:{media_type};base64,{data}" + + def _write_data_url_to_source(source: dict, data_url: str) -> None: + header, _, data = data_url.partition(",") + media_type = "image/jpeg" + if header.startswith("data:"): + candidate = header[len("data:"):].split(";", 1)[0].strip() + if candidate.startswith("image/"): + media_type = candidate + source["type"] = "base64" + source["media_type"] = media_type + source["data"] = data + for msg in api_messages: if not isinstance(msg, dict): continue @@ -974,6 +997,16 @@ def try_shrink_image_parts_in_messages( if not isinstance(part, dict): continue ptype = part.get("type") + if ptype == "image": + source = part.get("source") + url = _source_to_data_url(source) + resized, unshrinkable = _shrink_data_url(url or "") + if resized and isinstance(source, dict): + _write_data_url_to_source(source, resized) + changed_count += 1 + elif unshrinkable: + unshrinkable_oversized += 1 + continue if ptype not in {"image_url", "input_image"}: continue image_value = part.get("image_url") diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index 91496d72040..3a013503110 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -122,10 +122,14 @@ def finalize_turn( ) # Determine if conversation completed successfully + normal_text_response = str(_turn_exit_reason).startswith("text_response(") completed = ( final_response is not None - and api_call_count < agent.max_iterations and not failed + and ( + api_call_count < agent.max_iterations + or normal_text_response + ) ) # Post-loop cleanup must never lose the response. Trajectory save, diff --git a/apps/desktop/src/app/settings/computer-use-panel.tsx b/apps/desktop/src/app/settings/computer-use-panel.tsx new file mode 100644 index 00000000000..ada5c08e3ad --- /dev/null +++ b/apps/desktop/src/app/settings/computer-use-panel.tsx @@ -0,0 +1,239 @@ +import { useCallback, useEffect, useRef, useState } from 'react' + +import { Button } from '@/components/ui/button' +import { getActionStatus, getComputerUseStatus, grantComputerUsePermissions } from '@/hermes' +import { AlertTriangle, Check, ExternalLink, Loader2, RefreshCw, X } from '@/lib/icons' +import { upsertDesktopActionTask } from '@/store/activity' +import { notify, notifyError } from '@/store/notifications' +import type { ComputerUseStatus } from '@/types/hermes' + +import { Pill } from './primitives' + +interface ComputerUsePanelProps { + /** Re-read the parent toolset list after a permission/install change so the + * "Configured / Needs keys" pill stays in sync. */ + onConfiguredChange?: () => void +} + +// Per-OS one-liner shown when there's no TCC grant flow (Windows/Linux). macOS +// drives the permission rows instead, so it has no entry here. +const PLATFORM_NOTE: Record = { + linux: 'Drives your desktop via the X11/XWayland accessibility stack — no permission prompt.', + win32: 'First run may trigger a Windows SmartScreen prompt for the cua-driver UIAccess worker — allow it.' +} + +function tone(granted: boolean | null) { + return granted === true ? 'primary' : 'muted' +} + +function GrantIcon({ granted }: { granted: boolean | null }) { + const Icon = granted === true ? Check : granted === false ? X : AlertTriangle + + return +} + +function PermissionRow({ granted, label, hint }: { granted: boolean | null; label: string; hint: string }) { + return ( +
+
+ {label} +

{hint}

+
+ + + {granted === true ? 'Granted' : granted === false ? 'Not granted' : 'Unknown'} + +
+ ) +} + +/** + * Cross-platform Computer Use preflight card. + * + * cua-driver runs on macOS, Windows, and Linux, but readiness differs: macOS + * needs two TCC grants (Accessibility + Screen Recording) that attach to + * cua-driver's own `com.trycua.driver` identity — not Hermes — and are + * requested via `cua-driver permissions grant` (dialog attributed to + * CuaDriver). Windows/Linux have no TCC toggles, so readiness is driver health + * from `cua-driver doctor`. The backend folds both into one `ready` signal. + * + * Binary install/upgrade stays in the cua-driver provider's post-setup runner + * below this card (the generic ToolsetConfigPanel). + */ +export function ComputerUsePanel({ onConfiguredChange }: ComputerUsePanelProps) { + const [status, setStatus] = useState(null) + const [loading, setLoading] = useState(true) + const [granting, setGranting] = useState(false) + const activeRef = useRef(false) + + const refresh = useCallback(async () => { + try { + setStatus(await getComputerUseStatus()) + } catch (err) { + notifyError(err, 'Could not read Computer Use status') + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + activeRef.current = true + void refresh() + + return () => void (activeRef.current = false) + }, [refresh]) + + const grant = useCallback(async () => { + setGranting(true) + + try { + const started = await grantComputerUsePermissions() + + if (!started.ok) { + notifyError(new Error('spawn failed'), 'Could not request permissions') + + return + } + + notify({ + kind: 'info', + title: 'Approve in System Settings', + message: 'macOS will show a permission dialog attributed to CuaDriver. Approve it, then return here.' + }) + + // The driver waits for the user to flip the switch — poll until it exits. + for (let attempt = 0; attempt < 150 && activeRef.current; attempt += 1) { + await new Promise(resolve => window.setTimeout(resolve, 1500)) + + if (!activeRef.current) { + break + } + + const polled = await getActionStatus(started.name, 200) + upsertDesktopActionTask(polled) + + if (!polled.running) { + break + } + } + + if (activeRef.current) { + await refresh() + onConfiguredChange?.() + } + } catch (err) { + if (activeRef.current) { + notifyError(err, 'Could not request permissions') + } + } finally { + if (activeRef.current) { + setGranting(false) + } + } + }, [onConfiguredChange, refresh]) + + if (loading) { + return ( +
+ + Checking Computer Use status… +
+ ) + } + + if (!status) { + return null + } + + if (!status.platform_supported) { + return ( +

+ Computer Use isn't supported on this platform ({status.platform}). +

+ ) + } + + if (!status.installed) { + return ( +

+ Install the cua-driver backend below to drive this machine. + {status.can_grant && ' Then grant Accessibility and Screen Recording here.'} +

+ ) + } + + const failingChecks = status.checks.filter(c => c.status !== 'ok') + + return ( +
+
+
+ {status.can_grant ? ( +

+ Grants attach to CuaDriver's own identity (com.trycua.driver), not Hermes — so the dialog is + attributed to the process that drives your Mac. +

+ ) : ( +

{PLATFORM_NOTE[status.platform] ?? ''}

+ )} + {status.version &&

{status.version}

} +
+ +
+ + {status.can_grant ? ( + <> + + + + ) : ( +
+ Driver health + + + {status.ready === true ? 'Ready' : status.ready === false ? 'Not ready' : 'Unknown'} + +
+ )} + + {failingChecks.map(c => ( +

+ + {c.label}: {c.message} +

+ ))} + + {status.error && ( +

+ + {status.error} +

+ )} + + {status.ready ? ( +
+ + Computer Use is ready. Ask the agent to capture an app and click around. +
+ ) : ( + status.can_grant && ( + + ) + )} +
+ ) +} diff --git a/apps/desktop/src/app/skills/index.tsx b/apps/desktop/src/app/skills/index.tsx index 716f0181f12..90aa4a24357 100644 --- a/apps/desktop/src/app/skills/index.tsx +++ b/apps/desktop/src/app/skills/index.tsx @@ -17,6 +17,7 @@ import { useRefreshHotkey } from '../hooks/use-refresh-hotkey' import { useRouteEnumParam } from '../hooks/use-route-enum-param' import { PAGE_INSET_X } from '../layout-constants' import { PageSearchShell } from '../page-search-shell' +import { ComputerUsePanel } from '../settings/computer-use-panel' import { asText, includesQuery, prettyName, toolNames, toolsetDisplayLabel } from '../settings/helpers' import { ToolsetConfigPanel } from '../settings/toolset-config-panel' import type { SetStatusbarItemGroup } from '../shell/statusbar-controls' @@ -334,6 +335,9 @@ export function SkillsView({ setStatusbarItemGroup: _setStatusbarItemGroup, ...p ))} )} + {expanded && toolset.name === 'computer_use' && ( + + )} {expanded && } ) diff --git a/apps/desktop/src/components/assistant-ui/thread-timeline-data.test.ts b/apps/desktop/src/components/assistant-ui/thread-timeline-data.test.ts new file mode 100644 index 00000000000..a3cc48da56a --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/thread-timeline-data.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' + +import { activeTimelineIndex, deriveTimelineEntries, timelinePreview } from './thread-timeline-data' + +describe('timelinePreview', () => { + it('collapses whitespace to a single line', () => { + expect(timelinePreview('hello\n\n world\tagain')).toBe('hello world again') + }) + + it('truncates with an ellipsis past the limit', () => { + const out = timelinePreview('abcdefghij', 5) + expect(out).toBe('abcd…') + expect(out.length).toBe(5) + }) +}) + +describe('deriveTimelineEntries', () => { + it('keeps non-empty user prompts in order', () => { + expect( + deriveTimelineEntries([ + { id: 'u1', role: 'user', text: 'first' }, + { id: 'a1', role: 'assistant', text: 'answer' }, + { id: 'u2', role: 'user', text: ' second ' } + ]) + ).toEqual([ + { id: 'u1', preview: 'first' }, + { id: 'u2', preview: 'second' } + ]) + }) + + it('drops blanks and background-process notifications', () => { + expect( + deriveTimelineEntries([ + { id: 'u1', role: 'user', text: ' ' }, + { id: 'u2', role: 'user', text: '[IMPORTANT: Background process 123 finished]' }, + { id: 'u3', role: 'user', text: 'real prompt' } + ]).map(e => e.id) + ).toEqual(['u3']) + }) +}) + +describe('activeTimelineIndex', () => { + it('returns the last prompt scrolled to or above the top edge', () => { + expect(activeTimelineIndex([-400, -10, 320])).toBe(1) + }) + + it('falls back to the first rendered entry', () => { + expect(activeTimelineIndex([null, 120, 480])).toBe(1) + expect(activeTimelineIndex([null, null])).toBe(0) + }) +}) diff --git a/apps/desktop/src/components/assistant-ui/thread-timeline-data.ts b/apps/desktop/src/components/assistant-ui/thread-timeline-data.ts new file mode 100644 index 00000000000..e52d1d7c780 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/thread-timeline-data.ts @@ -0,0 +1,75 @@ +// Pure timeline helpers — no React/DOM; tested in thread-timeline-data.test.ts. + +export interface TimelineSourceMessage { + id: string + role: string + text: string +} + +export interface TimelineEntry { + id: string + preview: string +} + +// Injected as user messages for alternation; not human prompts (thread.tsx). +const PROCESS_NOTIFICATION_RE = /^\[IMPORTANT: Background process [\s\S]*\]$/ + +const PREVIEW_MAX = 120 + +export function timelinePreview(text: string, max: number = PREVIEW_MAX): string { + const collapsed = text.replace(/\s+/g, ' ').trim() + + if (collapsed.length <= max) { + return collapsed + } + + return `${collapsed.slice(0, max - 1).trimEnd()}…` +} + +export function deriveTimelineEntries(messages: readonly TimelineSourceMessage[]): TimelineEntry[] { + const entries: TimelineEntry[] = [] + + for (const message of messages) { + if (message.role !== 'user') { + continue + } + + const text = message.text.trim() + + if (!text || PROCESS_NOTIFICATION_RE.test(text)) { + continue + } + + entries.push({ id: message.id, preview: timelinePreview(text) }) + } + + return entries +} + +/** Last user prompt at/above the viewport top (with slack); else first rendered. */ +export function activeTimelineIndex(offsets: readonly (number | null)[], slack: number = 8): number { + let active = -1 + let firstRendered = -1 + + for (let i = 0; i < offsets.length; i++) { + const offset = offsets[i] + + if (offset == null) { + continue + } + + if (firstRendered === -1) { + firstRendered = i + } + + if (offset <= slack) { + active = i + } + } + + if (active !== -1) { + return active + } + + return firstRendered === -1 ? 0 : firstRendered +} diff --git a/apps/desktop/src/components/assistant-ui/thread-timeline.tsx b/apps/desktop/src/components/assistant-ui/thread-timeline.tsx new file mode 100644 index 00000000000..e330cb6d755 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/thread-timeline.tsx @@ -0,0 +1,272 @@ +import { useAuiState } from '@assistant-ui/react' +import { type FC, useCallback, useEffect, useMemo, useRef, useState } from 'react' + +import { composerPanelCard } from '@/components/chat/composer-dock' +import { triggerHaptic } from '@/lib/haptics' +import { cn } from '@/lib/utils' +import { setPaneHoverRevealSuppressed } from '@/store/panes' + +import { + activeTimelineIndex, + deriveTimelineEntries, + type TimelineEntry, + type TimelineSourceMessage +} from './thread-timeline-data' + +const MIN_ENTRIES = 4 +const VIEWPORT = '[data-slot="aui_thread-viewport"]' +const HOVER_CLOSE_MS = 140 + +const ROW_CLASS = + 'relative flex w-full min-w-0 max-w-full cursor-pointer select-none overflow-hidden rounded-md px-2 py-1 text-left outline-hidden transition-colors duration-100 ease-out hover:bg-(--ui-row-hover-background) hover:transition-none' + +const POPOVER_SHELL = cn( + 'absolute right-full top-1/2 z-50 mr-1.5 max-h-[min(22rem,calc(100vh-8rem))] w-80 max-w-[min(20rem,calc(100vw-2rem))] -translate-y-1/2 overflow-x-hidden overflow-y-auto overscroll-contain p-1 text-popover-foreground transition-[opacity,transform] duration-100 ease-out group-hover/timeline:transition-none', + composerPanelCard, + // Solid fill — composerPanelCard is deliberately translucent; without this, + // directive chips in the transcript bleed through and look like popover overflow. + 'bg-(--composer-fill)' +) + +function userPromptText(content: unknown): string { + if (typeof content === 'string') { + return content + } + + if (!Array.isArray(content)) { + return '' + } + + let out = '' + + for (const part of content) { + if (typeof part === 'string') { + out += part + + continue + } + + if (!part || typeof part !== 'object') { + continue + } + + const row = part as { text?: unknown; type?: unknown } + + if ((!row.type || row.type === 'text') && typeof row.text === 'string') { + out += row.text + } + } + + return out +} + +function scrollToPrompt(id: string) { + const viewport = document.querySelector(VIEWPORT) + const node = viewport?.querySelector(`[data-message-id="${CSS.escape(id)}"]`) + + if (!viewport || !node) { + return + } + + const top = viewport.scrollTop + (node.getBoundingClientRect().top - viewport.getBoundingClientRect().top) - 8 + + triggerHaptic('selection') + viewport.scrollTo({ behavior: 'smooth', top: Math.max(0, top) }) +} + +/** Right-edge prompt rail — hover previews, click to jump. ≥4 user turns only. */ +export const ThreadTimeline: FC = () => { + const sourceSignature = useAuiState(s => { + const rows: TimelineSourceMessage[] = [] + + for (const message of s.thread.messages) { + if (message.role !== 'user') { + continue + } + + rows.push({ id: message.id, role: 'user', text: userPromptText(message.content) }) + } + + return JSON.stringify(rows) + }) + + const entries = useMemo( + () => deriveTimelineEntries(JSON.parse(sourceSignature) as TimelineSourceMessage[]), + [sourceSignature] + ) + + const [activeIndex, setActiveIndex] = useState(0) + const [hoverIndex, setHoverIndex] = useState(null) + const [open, setOpen] = useState(false) + const closeTimerRef = useRef(undefined) + + const keepOpen = useCallback(() => { + window.clearTimeout(closeTimerRef.current) + setPaneHoverRevealSuppressed(true) + setOpen(true) + }, []) + + const closeSoon = useCallback(() => { + window.clearTimeout(closeTimerRef.current) + setHoverIndex(null) + setPaneHoverRevealSuppressed(false) + closeTimerRef.current = window.setTimeout(() => setOpen(false), HOVER_CLOSE_MS) + }, []) + + useEffect( + () => () => { + window.clearTimeout(closeTimerRef.current) + setPaneHoverRevealSuppressed(false) + }, + [] + ) + + useEffect(() => { + if (entries.length < MIN_ENTRIES) { + setPaneHoverRevealSuppressed(false) + } + }, [entries.length]) + + useEffect(() => { + const viewport = document.querySelector(VIEWPORT) + + if (!viewport || entries.length === 0) { + return + } + + let raf = 0 + + const compute = () => { + raf = 0 + + const top = viewport.getBoundingClientRect().top + + const offsets = entries.map(entry => { + const node = viewport.querySelector(`[data-message-id="${CSS.escape(entry.id)}"]`) + + return node ? node.getBoundingClientRect().top - top : null + }) + + const next = activeTimelineIndex(offsets) + + setActiveIndex(prev => (prev === next ? prev : next)) + } + + const onScroll = () => { + if (!raf) { + raf = requestAnimationFrame(compute) + } + } + + compute() + viewport.addEventListener('scroll', onScroll, { passive: true }) + + return () => { + viewport.removeEventListener('scroll', onScroll) + + if (raf) { + cancelAnimationFrame(raf) + } + } + }, [entries]) + + if (entries.length < MIN_ENTRIES) { + return null + } + + return ( +
+ + +
+ ) +} + +const TimelinePopover: FC<{ + activeIndex: number + entries: TimelineEntry[] + hoverIndex: number | null + onHover: (index: number) => void + onJump: (id: string) => void + open: boolean +}> = ({ activeIndex, entries, hoverIndex, onHover, onJump, open }) => ( +
+ {entries.map((entry, index) => { + const hovered = index === hoverIndex + const active = index === activeIndex + + return ( + + ) + })} +
+) + +const TimelineTicks: FC<{ + activeIndex: number + entries: TimelineEntry[] + onHover: (index: number) => void + onJump: (id: string) => void +}> = ({ activeIndex, entries, onHover, onJump }) => ( +
+ {entries.map((entry, index) => ( + + ))} +
+) diff --git a/apps/desktop/src/components/assistant-ui/thread.tsx b/apps/desktop/src/components/assistant-ui/thread.tsx index 1ac97c200ca..6057307dec3 100644 --- a/apps/desktop/src/components/assistant-ui/thread.tsx +++ b/apps/desktop/src/components/assistant-ui/thread.tsx @@ -64,6 +64,7 @@ import { ClarifyTool } from '@/components/assistant-ui/clarify-tool' import { DirectiveContent, hermesDirectiveFormatter } from '@/components/assistant-ui/directive-text' import { MarkdownText, MarkdownTextContent } from '@/components/assistant-ui/markdown-text' import { ThreadMessageList } from '@/components/assistant-ui/thread-list' +import { ThreadTimeline } from '@/components/assistant-ui/thread-timeline' import { ToolFallback, ToolGroupSlot } from '@/components/assistant-ui/tool-fallback' import { TooltipIconButton } from '@/components/assistant-ui/tooltip-icon-button' import { UserMessageText } from '@/components/assistant-ui/user-message-text' @@ -212,6 +213,7 @@ export const Thread: FC<{ sessionKey={sessionKey} /> {loading === 'session' && } + ) } @@ -797,7 +799,15 @@ function messageAttachmentRefs(value: unknown): string[] { return value.every(ref => typeof ref === 'string') ? value : EMPTY_ATTACHMENT_REFS } -function StickyHumanMessageContainer({ attachments, children }: { attachments?: ReactNode; children: ReactNode }) { +function StickyHumanMessageContainer({ + attachments, + children, + messageId +}: { + attachments?: ReactNode + children: ReactNode + messageId?: string +}) { return ( // Fragment, not a wrapper: a wrapping element becomes the sticky's // containing block (it'd stick within its own height = never). The bubble @@ -806,6 +816,7 @@ function StickyHumanMessageContainer({ attachments, children }: { attachments?: <>
@@ -990,6 +1001,7 @@ const UserMessage: FC<{ return ( (null) // Keyboard (mod+b / mod+j) pins the reveal open while collapsed; hover is CSS. @@ -378,7 +379,10 @@ export function Pane({ >