mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix: ground compression task snapshot
This commit is contained in:
parent
960abf73a0
commit
761a0b124e
2 changed files with 102 additions and 9 deletions
|
|
@ -299,6 +299,7 @@ _FALLBACK_TURN_MAX_CHARS = 700
|
||||||
_AUTO_FOCUS_MAX_TURNS = 3
|
_AUTO_FOCUS_MAX_TURNS = 3
|
||||||
_AUTO_FOCUS_TURN_MAX_CHARS = 260
|
_AUTO_FOCUS_TURN_MAX_CHARS = 260
|
||||||
_AUTO_FOCUS_MAX_CHARS = 700
|
_AUTO_FOCUS_MAX_CHARS = 700
|
||||||
|
_ACTIVE_TASK_MAX_CHARS = 1400
|
||||||
# Keep a short run of recent messages verbatim even when the token budget is
|
# Keep a short run of recent messages verbatim even when the token budget is
|
||||||
# already exhausted. The public ``protect_last_n`` default is intentionally
|
# already exhausted. The public ``protect_last_n`` default is intentionally
|
||||||
# high for small/light tails, but using all 20 as a hard floor here would bring
|
# high for small/light tails, but using all 20 as a hard floor here would bring
|
||||||
|
|
@ -322,6 +323,9 @@ _PATH_MENTION_RE = re.compile(r"(?:/|~/?|[A-Za-z]:\\)[^\s`'\")\]}<>]+")
|
||||||
# the summary, the downstream model may re-emit it as an active directive on
|
# the summary, the downstream model may re-emit it as an active directive on
|
||||||
# the next turn, triggering bogus attachment sends (#14665).
|
# the next turn, triggering bogus attachment sends (#14665).
|
||||||
_MEDIA_DIRECTIVE_RE = re.compile(r"MEDIA:\S+")
|
_MEDIA_DIRECTIVE_RE = re.compile(r"MEDIA:\S+")
|
||||||
|
_HISTORICAL_TASK_SECTION_RE = re.compile(
|
||||||
|
rf"(?ms)^{re.escape(HISTORICAL_TASK_HEADING)}\s*\n.*?(?=^## |\Z)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _dedupe_append(items: list[str], value: str, *, limit: int) -> None:
|
def _dedupe_append(items: list[str], value: str, *, limit: int) -> None:
|
||||||
|
|
@ -2146,9 +2150,9 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb
|
||||||
_template_sections = f"""{HISTORICAL_TASK_HEADING}
|
_template_sections = f"""{HISTORICAL_TASK_HEADING}
|
||||||
[THE SINGLE MOST IMPORTANT FIELD. Capture the user's most recent unfulfilled
|
[THE SINGLE MOST IMPORTANT FIELD. Capture the user's most recent unfulfilled
|
||||||
input verbatim — the exact words they used. This includes:
|
input verbatim — the exact words they used. This includes:
|
||||||
- Explicit task assignments ("refactor the auth module")
|
- Explicit task assignments ("<specific user task>")
|
||||||
- Questions awaiting an answer ("waarom staat X op Y?", "wat zijn de volgende stappen?")
|
- Questions awaiting an answer ("<specific user question>")
|
||||||
- Decisions awaiting input ("optie A of B?")
|
- Decisions awaiting input ("<option A or B?>")
|
||||||
- Ongoing discussions where the assistant owes the next substantive reply
|
- Ongoing discussions where the assistant owes the next substantive reply
|
||||||
A conversation where the user just asked a question IS an active task — the
|
A conversation where the user just asked a question IS an active task — the
|
||||||
task is "answer that question with full context". Do NOT write "None" merely
|
task is "answer that question with full context". Do NOT write "None" merely
|
||||||
|
|
@ -2156,15 +2160,15 @@ because the user did not issue an imperative command; reserve "None" for the
|
||||||
rare case where the last exchange was fully resolved and the user said
|
rare case where the last exchange was fully resolved and the user said
|
||||||
something like "thanks, that's all".
|
something like "thanks, that's all".
|
||||||
If multiple items are outstanding, list only the ones NOT yet completed.
|
If multiple items are outstanding, list only the ones NOT yet completed.
|
||||||
Continuation should pick up exactly here. Examples:
|
This historical snapshot must identify the latest unresolved user input precisely. Examples:
|
||||||
"User asked: 'Now refactor the auth module to use JWT instead of sessions'"
|
"User asked: '<exact latest user request>'"
|
||||||
"User asked: 'Waarom stond provider ineens op openrouter?' — needs investigation + answer"
|
"User asked: '<exact latest user question>' — needs investigation + answer"
|
||||||
"User chose option A; awaiting implementation of step 2"
|
"User chose <option>; awaiting implementation of <specific next step>"
|
||||||
If the user's most recent message was a reverse signal (stop, undo, roll
|
If the user's most recent message was a reverse signal (stop, undo, roll
|
||||||
back, never mind, just verify, change of topic) that supersedes earlier
|
back, never mind, just verify, change of topic) that supersedes earlier
|
||||||
work, write the reverse signal verbatim and DO NOT carry forward the
|
work, write the reverse signal verbatim and DO NOT carry forward the
|
||||||
cancelled task. Example: "User asked: 'Stop the i18n refactor and just
|
cancelled task. Example: "User asked: '<exact reverse signal>' — earlier
|
||||||
verify the current diff' — earlier i18n in-flight work is cancelled."
|
in-flight work is cancelled."
|
||||||
If no outstanding task exists, write "None."]
|
If no outstanding task exists, write "None."]
|
||||||
|
|
||||||
## Goal
|
## Goal
|
||||||
|
|
@ -2326,6 +2330,7 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||||
# Redact the summary output as well — the summarizer LLM may
|
# Redact the summary output as well — the summarizer LLM may
|
||||||
# ignore prompt instructions and echo back secrets verbatim.
|
# ignore prompt instructions and echo back secrets verbatim.
|
||||||
summary = redact_sensitive_text(content.strip())
|
summary = redact_sensitive_text(content.strip())
|
||||||
|
summary = self._ground_historical_task_snapshot(summary, turns_to_summarize)
|
||||||
# Store for iterative updates on next compaction
|
# Store for iterative updates on next compaction
|
||||||
self._previous_summary = summary
|
self._previous_summary = summary
|
||||||
self._clear_compression_failure_cooldown()
|
self._clear_compression_failure_cooldown()
|
||||||
|
|
@ -2597,6 +2602,54 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||||
focus = focus[: _AUTO_FOCUS_MAX_CHARS - 1].rstrip() + "…"
|
focus = focus[: _AUTO_FOCUS_MAX_CHARS - 1].rstrip() + "…"
|
||||||
return focus
|
return focus
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _latest_user_task_snapshot(
|
||||||
|
cls,
|
||||||
|
messages: List[Dict[str, Any]],
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Return a deterministic task-snapshot line from the newest real user turn.
|
||||||
|
|
||||||
|
The LLM summarizer is allowed to compress prose, but it must not invent
|
||||||
|
the "what is the active task?" anchor from a prompt example or stale
|
||||||
|
prior summary. This helper extracts the anchor locally from the exact
|
||||||
|
compacted turns so the summary can be grounded before it becomes live
|
||||||
|
context.
|
||||||
|
"""
|
||||||
|
for msg in reversed(messages):
|
||||||
|
if msg.get("role") != "user":
|
||||||
|
continue
|
||||||
|
content = msg.get("content")
|
||||||
|
if cls._is_context_summary_content(content):
|
||||||
|
continue
|
||||||
|
text = redact_sensitive_text(_content_text_for_contains(content).strip())
|
||||||
|
if not text:
|
||||||
|
continue
|
||||||
|
text = re.sub(r"\s+", " ", text)
|
||||||
|
if len(text) > _ACTIVE_TASK_MAX_CHARS:
|
||||||
|
text = text[: _ACTIVE_TASK_MAX_CHARS - 15].rstrip() + " ...[truncated]"
|
||||||
|
return (
|
||||||
|
f"User asked (deterministic, from compacted turns): {text!r}\n"
|
||||||
|
"Historical only; newer protected-tail messages after this summary win."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _ground_historical_task_snapshot(
|
||||||
|
cls,
|
||||||
|
summary: str,
|
||||||
|
messages: List[Dict[str, Any]],
|
||||||
|
) -> str:
|
||||||
|
"""Force the task snapshot section to match a real user turn when possible."""
|
||||||
|
snapshot = cls._latest_user_task_snapshot(messages)
|
||||||
|
if not snapshot:
|
||||||
|
return summary
|
||||||
|
|
||||||
|
body = cls._strip_summary_prefix(summary)
|
||||||
|
replacement = f"{HISTORICAL_TASK_HEADING}\n{snapshot}"
|
||||||
|
if _HISTORICAL_TASK_SECTION_RE.search(body):
|
||||||
|
return _HISTORICAL_TASK_SECTION_RE.sub(replacement, body, count=1)
|
||||||
|
return f"{replacement}\n\n{body}".strip()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _find_latest_context_summary(
|
def _find_latest_context_summary(
|
||||||
cls,
|
cls,
|
||||||
|
|
|
||||||
|
|
@ -761,9 +761,49 @@ class TestNonStringContent:
|
||||||
assert "Do NOT respond" not in prompt
|
assert "Do NOT respond" not in prompt
|
||||||
assert "DIFFERENT assistant" not in prompt
|
assert "DIFFERENT assistant" not in prompt
|
||||||
assert "different assistant" not in prompt
|
assert "different assistant" not in prompt
|
||||||
|
assert "refactor the auth module" not in prompt
|
||||||
|
assert "JWT instead of sessions" not in prompt
|
||||||
assert "Treat the conversation turns below as source material" in prompt
|
assert "Treat the conversation turns below as source material" in prompt
|
||||||
assert "structured checkpoint summary" in prompt
|
assert "structured checkpoint summary" in prompt
|
||||||
|
|
||||||
|
def test_summary_task_snapshot_is_grounded_to_latest_user_turn(self):
|
||||||
|
"""Regression for a copied prompt example becoming the active task.
|
||||||
|
|
||||||
|
A real #26 run showed the summarizer emitting the old template example
|
||||||
|
"Now refactor the auth module to use JWT instead of sessions". The
|
||||||
|
compacted turns did not contain that request, so the summary must
|
||||||
|
replace it with the deterministic latest user turn before the handoff
|
||||||
|
becomes live context.
|
||||||
|
"""
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.choices = [MagicMock()]
|
||||||
|
mock_response.choices[0].message.content = """## Historical Task Snapshot
|
||||||
|
User asked: 'Now refactor the auth module to use JWT instead of sessions'
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
Continue the task.
|
||||||
|
|
||||||
|
## Completed Actions
|
||||||
|
None.
|
||||||
|
"""
|
||||||
|
|
||||||
|
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||||
|
c = ContextCompressor(model="test", quiet_mode=True)
|
||||||
|
|
||||||
|
latest = "RUN_LONG_REAL_26_SCORE_V3B_WITH_FACTUALITY_SMOKE"
|
||||||
|
messages = [
|
||||||
|
{"role": "user", "content": latest},
|
||||||
|
{"role": "assistant", "content": "I will inspect the loop harness."},
|
||||||
|
]
|
||||||
|
|
||||||
|
with patch("agent.context_compressor.call_llm", return_value=mock_response):
|
||||||
|
summary = c._generate_summary(messages)
|
||||||
|
|
||||||
|
assert "refactor the auth module" not in summary
|
||||||
|
assert "JWT instead of sessions" not in summary
|
||||||
|
assert latest in summary
|
||||||
|
assert "deterministic, from compacted turns" in summary
|
||||||
|
|
||||||
def test_summary_call_passes_live_main_runtime(self):
|
def test_summary_call_passes_live_main_runtime(self):
|
||||||
mock_response = MagicMock()
|
mock_response = MagicMock()
|
||||||
mock_response.choices = [MagicMock()]
|
mock_response.choices = [MagicMock()]
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue