mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-15 14:22:43 +00:00
feat(moa): opt-in full-turn trace persistence to JSONL (#56101)
Adds moa.save_traces (default off). When on, every MoA turn that runs the reference fan-out appends one JSON line to <hermes_home>/moa-traces/<session_id>.jsonl capturing the TRUE FULL turn: each reference model's exact input messages (system advisory prompt + full advisory view, not the truncated display preview) + full output + usage + per-advisor cost, and the aggregator's exact input (including the injected reference-context guidance block) + output. Lets MoA runs be audited and improved offline — what every model saw, said, and cost. - agent/moa_trace.py: config-gated JSONL writer, profile-aware path via get_hermes_home(), best-effort (never breaks a turn), moa.trace_dir override. - agent/moa_loop.py: _RefAccounting now carries full input/output/model/ provider/temperature; create() stashes the full turn on a cache MISS (once per turn, never on the cache-HIT repeat iterations); non-streaming aggregator output captured inline, streaming marked + pointed at the session assistant message. consume_and_save_trace(session_id) flushes it. - agent/conversation_loop.py: flushes the trace with the live session_id right after MoA usage consumption. No-op for non-MoA clients. - hermes_cli/config.py: moa.save_traces + moa.trace_dir defaults. Traces are a side channel — NOT the messages table, never in replay, safe to delete. Off by default; only overhead when off is one config read on a MoA cache-MISS turn. Tests: full-trace-when-enabled (per-ref input+output+cost, aggregator input-with-guidance + output), nothing-when-disabled. Live E2E through run_conversation confirmed the loop wiring writes the file.
This commit is contained in:
Teknium
GitHub
parent
5f7deeba84
commit
2e8748ed22
5 changed files with 437 additions and 8 deletions
|
|
@ -28,8 +28,8 @@ _MAX_REFERENCE_WORKERS = 8
|
|||
|
||||
|
||||
class _RefAccounting:
|
||||
"""Per-reference token usage + estimated cost, carried as the third slot
|
||||
of a reference-output tuple.
|
||||
"""Per-reference token usage + estimated cost + full trace, carried as the
|
||||
third slot of a reference-output tuple.
|
||||
|
||||
Kept as a tiny object (not a bare CanonicalUsage) because an advisor may
|
||||
run on a different model/provider than the aggregator, so its cost MUST be
|
||||
|
|
@ -37,15 +37,48 @@ class _RefAccounting:
|
|||
aggregator's usage and pricing the sum at the aggregator's rate would
|
||||
misprice every advisor. ``usage`` feeds accurate token counts;
|
||||
``cost_usd`` feeds accurate cost.
|
||||
|
||||
``messages`` / ``output`` / ``model`` / ``provider`` / ``temperature``
|
||||
carry the FULL reference input and output for trace persistence (the
|
||||
display ``text`` is a truncated preview and is not enough to audit what an
|
||||
advisor actually saw). They are only populated when tracing is on; they add
|
||||
negligible cost otherwise.
|
||||
"""
|
||||
|
||||
__slots__ = ("usage", "cost_usd", "cost_status", "cost_source")
|
||||
__slots__ = (
|
||||
"usage",
|
||||
"cost_usd",
|
||||
"cost_status",
|
||||
"cost_source",
|
||||
"messages",
|
||||
"output",
|
||||
"model",
|
||||
"provider",
|
||||
"temperature",
|
||||
)
|
||||
|
||||
def __init__(self, usage: Any, cost_usd: Any = None, cost_status: str | None = None, cost_source: str | None = None):
|
||||
def __init__(
|
||||
self,
|
||||
usage: Any,
|
||||
cost_usd: Any = None,
|
||||
cost_status: str | None = None,
|
||||
cost_source: str | None = None,
|
||||
*,
|
||||
messages: Any = None,
|
||||
output: str | None = None,
|
||||
model: str | None = None,
|
||||
provider: str | None = None,
|
||||
temperature: Any = None,
|
||||
):
|
||||
self.usage = usage
|
||||
self.cost_usd = cost_usd
|
||||
self.cost_status = cost_status
|
||||
self.cost_source = cost_source
|
||||
self.messages = messages
|
||||
self.output = output
|
||||
self.model = model
|
||||
self.provider = provider
|
||||
self.temperature = temperature
|
||||
|
||||
# Per-tool-result character budget for the advisory reference view. Tool
|
||||
# results can be huge (a full diff, a 5000-line file dump); replaying them
|
||||
|
|
@ -219,11 +252,29 @@ def _run_reference(
|
|||
cost_source = cost.source
|
||||
except Exception: # pragma: no cover - defensive
|
||||
pass
|
||||
acct = _RefAccounting(usage, cost_usd, cost_status, cost_source)
|
||||
return label, _extract_text(response) or "(empty response)", acct
|
||||
_output_text = _extract_text(response) or "(empty response)"
|
||||
acct = _RefAccounting(
|
||||
usage,
|
||||
cost_usd,
|
||||
cost_status,
|
||||
cost_source,
|
||||
messages=messages,
|
||||
output=_output_text,
|
||||
model=slot.get("model"),
|
||||
provider=runtime.get("provider") or slot.get("provider"),
|
||||
temperature=temperature,
|
||||
)
|
||||
return label, _output_text, acct
|
||||
except Exception as exc:
|
||||
logger.warning("MoA reference model %s failed: %s", label, exc)
|
||||
return label, f"[failed: {exc}]", _RefAccounting(CanonicalUsage())
|
||||
return label, f"[failed: {exc}]", _RefAccounting(
|
||||
CanonicalUsage(),
|
||||
messages=[{"role": "system", "content": _REFERENCE_SYSTEM_PROMPT}, *ref_messages],
|
||||
output=f"[failed: {exc}]",
|
||||
model=slot.get("model"),
|
||||
provider=runtime.get("provider") or slot.get("provider"),
|
||||
temperature=temperature,
|
||||
)
|
||||
|
||||
|
||||
def _run_references_parallel(
|
||||
|
|
@ -545,6 +596,10 @@ class MoAChatCompletions:
|
|||
|
||||
self._pending_reference_usage: Any = CanonicalUsage()
|
||||
self._pending_reference_cost: Any = None
|
||||
# Full-turn trace parts stashed on a cache-MISS create(), awaiting the
|
||||
# caller to stitch in the live session_id + resolved aggregator output
|
||||
# and flush to the trace file (only when moa.save_traces is on).
|
||||
self._pending_trace: Any = None
|
||||
|
||||
def consume_reference_usage(self) -> tuple[Any, Any]:
|
||||
"""Pop pending reference-fan-out usage + cost, resetting both to empty.
|
||||
|
|
@ -563,6 +618,37 @@ class MoAChatCompletions:
|
|||
self._pending_reference_cost = None
|
||||
return usage, cost
|
||||
|
||||
def consume_and_save_trace(self, session_id: Any = None) -> None:
|
||||
"""Flush the pending full-turn trace to disk, if one is pending.
|
||||
|
||||
No-op when tracing is off (``save_moa_turn`` checks the config), when
|
||||
there is no pending trace (a cache-HIT iteration ran no references), or
|
||||
when the aggregator input was never recorded. Clears the pending trace
|
||||
so a repeat consume cannot double-write. Best-effort — never raises.
|
||||
"""
|
||||
pending = self._pending_trace
|
||||
self._pending_trace = None
|
||||
if not pending or "aggregator_input_messages" not in pending:
|
||||
return
|
||||
try:
|
||||
from agent.moa_trace import save_moa_turn
|
||||
|
||||
agg_slot = pending.get("aggregator_slot") or {}
|
||||
save_moa_turn(
|
||||
session_id=session_id,
|
||||
preset_name=pending.get("preset", ""),
|
||||
reference_outputs=pending.get("reference_outputs", []),
|
||||
aggregator_label=pending.get("aggregator_label", ""),
|
||||
aggregator_model=agg_slot.get("model"),
|
||||
aggregator_provider=agg_slot.get("provider"),
|
||||
aggregator_temperature=pending.get("aggregator_temperature"),
|
||||
aggregator_input_messages=pending.get("aggregator_input_messages"),
|
||||
aggregator_output=pending.get("aggregator_output"),
|
||||
aggregator_streamed=bool(pending.get("aggregator_streamed")),
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - tracing must never break a turn
|
||||
logger.debug("MoA trace flush failed: %s", exc)
|
||||
|
||||
def _emit(self, event: str, **kwargs: Any) -> None:
|
||||
cb = self.reference_callback
|
||||
if cb is None:
|
||||
|
|
@ -618,6 +704,10 @@ class MoAChatCompletions:
|
|||
# advisor spend by the tool-iteration count, so pending is zero.
|
||||
self._pending_reference_usage = CanonicalUsage()
|
||||
self._pending_reference_cost = None
|
||||
# Likewise no trace on a cache HIT — the full turn was already
|
||||
# traced on the MISS that ran the references. A repeat iteration is
|
||||
# not a new MoA turn.
|
||||
self._pending_trace = None
|
||||
else:
|
||||
reference_outputs = _run_references_parallel(
|
||||
reference_models,
|
||||
|
|
@ -645,6 +735,17 @@ class MoAChatCompletions:
|
|||
_ref_cost = (_ref_cost or 0) + _acct.cost_usd
|
||||
self._pending_reference_usage = _ref_usage
|
||||
self._pending_reference_cost = _ref_cost
|
||||
# Stash the full reference fan-out for trace persistence. The
|
||||
# aggregator input/label are filled in below once agg_messages is
|
||||
# built; the aggregator OUTPUT is stitched in by the caller
|
||||
# (consume_and_save_trace) once the response resolves — the caller
|
||||
# holds the live session_id and the resolved aggregator response.
|
||||
self._pending_trace = {
|
||||
"preset": self.preset_name,
|
||||
"reference_outputs": list(reference_outputs),
|
||||
"aggregator_slot": aggregator,
|
||||
"aggregator_temperature": aggregator_temperature,
|
||||
}
|
||||
|
||||
# Surface each reference model's answer to the display BEFORE the
|
||||
# aggregator acts — once per turn (only on the iteration that
|
||||
|
|
@ -694,6 +795,12 @@ class MoAChatCompletions:
|
|||
raise RuntimeError("MoA aggregator cannot be another MoA preset")
|
||||
agg_kwargs = dict(api_kwargs)
|
||||
agg_kwargs["messages"] = agg_messages
|
||||
# Record the exact aggregator INPUT (incl. the injected reference
|
||||
# context) into the pending trace so a trace captures what the
|
||||
# aggregator actually saw, not a reconstruction.
|
||||
if self._pending_trace is not None:
|
||||
self._pending_trace["aggregator_input_messages"] = agg_messages
|
||||
self._pending_trace["aggregator_label"] = _slot_label(aggregator)
|
||||
# The aggregator is the acting model. Resolve its slot to the provider's
|
||||
# real runtime (base_url/api_key/api_mode) and call it through the same
|
||||
# request-building path any model uses — so per-model wire-format
|
||||
|
|
@ -720,7 +827,7 @@ class MoAChatCompletions:
|
|||
# actually governs the aggregator stream, not just call_llm's default.
|
||||
if api_kwargs.get("timeout") is not None:
|
||||
stream_kwargs["timeout"] = api_kwargs["timeout"]
|
||||
return call_llm(
|
||||
_agg_response = call_llm(
|
||||
task="moa_aggregator",
|
||||
messages=agg_messages,
|
||||
temperature=aggregator_temperature,
|
||||
|
|
@ -730,6 +837,22 @@ class MoAChatCompletions:
|
|||
**stream_kwargs,
|
||||
**_slot_runtime(aggregator),
|
||||
)
|
||||
# Non-streaming path (quiet mode / eval / subagents): the aggregator
|
||||
# output is available inline, so capture it into the pending trace now.
|
||||
# Streaming path: the aggregator's raw token stream is returned to the
|
||||
# consumer live and its acting output lands as the turn's assistant
|
||||
# message; the trace marks it streamed and points there.
|
||||
if self._pending_trace is not None:
|
||||
if stream:
|
||||
self._pending_trace["aggregator_streamed"] = True
|
||||
self._pending_trace["aggregator_output"] = None
|
||||
else:
|
||||
self._pending_trace["aggregator_streamed"] = False
|
||||
try:
|
||||
self._pending_trace["aggregator_output"] = _extract_text(_agg_response)
|
||||
except Exception: # pragma: no cover - defensive
|
||||
self._pending_trace["aggregator_output"] = None
|
||||
return _agg_response
|
||||
|
||||
|
||||
class MoAClient:
|
||||
|
|
@ -744,3 +867,10 @@ class MoAClient:
|
|||
usage without reaching into ``.chat.completions`` internals.
|
||||
"""
|
||||
return self.chat.completions.consume_reference_usage()
|
||||
|
||||
def consume_and_save_trace(self, session_id: Any = None) -> None:
|
||||
"""Flush the pending full-turn MoA trace via the completions facade.
|
||||
|
||||
No-op unless ``moa.save_traces`` is enabled and a turn is pending.
|
||||
"""
|
||||
return self.chat.completions.consume_and_save_trace(session_id)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue