feat(telemetry): write tel_spans — reconstructable run -> calls trace

Addresses the review on #51714: the trace/span layer was declared but unwired —
tel_spans was never written, call rows had no timestamp, and nothing set parent
lineage, so the store was metrics-only and couldn't reconstruct a trace.

Wire the span layer (keeping the praised star-schema shape):
  - New SpanEvent (span_id/trace_id/run_id/parent_span_id/name/kind/start_ns/end_ns)
    mapped into tel_spans via the emitter's _TABLE_COLUMNS.
  - The plugin mints a root span per run and, on each model/tool call, emits a
    SpanEvent (timing + parent = the run's root) keyed by the SAME span_id as the
    detail row, so tel_model_calls / tel_tool_calls JOIN to their span.
  - Call hooks fire on completion, so end_ns = now and start_ns is reconstructed
    from the measured latency/duration. The run's root span is emitted at finalize
    with the true run start/end.

Result: tel_spans is a connected, single-trace_id, run -> calls tree a desktop
waterfall (or any reader) can render directly, ordered by start_ns. Existing
metrics rows (tel_runs/model_calls/tool_calls) are unchanged.

OTLP: spans now flow to the exporter with their trace/parent/timing attributes.
The exporter still emits one OTel span per event rather than reconstructing OTel
SpanContexts into a connected trace tree; that projection is left for a follow-up
and the module docstring now says so plainly instead of over-claiming.

Adds test_spans_trace.py (connected-tree + detail-row JOIN) over the real dispatch
path. Accurate (pre-hook) start times, real OTLP SpanContexts, and subagent
cross-run lineage remain follow-ups.
This commit is contained in:
emozilla 2026-06-27 00:52:40 -04:00 committed by Victor Kyriazakos
parent 81a34156a5
commit 7c80b79bb0
5 changed files with 218 additions and 16 deletions

View file

@ -10,12 +10,15 @@ swallowed by core, and we additionally guard each callback so a telemetry bug ca
disturb a session. No content, no network: local telemetry only.
Hooks consumed:
on_session_start -> begin a run context (trace_id/run_id), buffer a run row
post_api_request -> one model_call event (tokens, latency, raw provider/model)
on_session_start -> begin a run context (trace_id/run_id + root span id)
post_api_request -> one model_call event + its timing span (tokens, latency)
api_request_error -> one error event
post_tool_call -> one tool_call event (raw tool name, duration, result class)
on_session_finalize -> finalize the run row (end_reason, counts, cost)
post_tool_call -> one tool_call event + its timing span (duration, result class)
on_session_finalize -> finalize the run row + emit the run's root span
subagent_start/stop -> (reserved) lineage markers
Each model/tool call emits a SpanEvent (timing + parent = the run's root span) keyed by
the same span_id as its detail row, so tel_spans reconstructs a run -> calls trace tree.
"""
from __future__ import annotations
@ -56,15 +59,18 @@ def _on_session_start(**kw: Any) -> None:
session_id = kw.get("session_id") or ""
platform = kw.get("platform") or kw.get("source") or ""
ctx = spans.start_run()
now = time.time_ns()
root_span_id = spans.new_span_id()
key = _run_key(session_id, kw.get("task_id"))
with _runs_lock:
_runs[key] = {
"run_id": ctx.run_id,
"trace_id": ctx.trace_id,
"root_span_id": root_span_id,
"session_id": session_id or None,
"entrypoint": _entrypoint_for(platform, kw.get("source")),
"platform": platform or None,
"start_ns": time.time_ns(),
"start_ns": now,
"model_call_count": 0,
"tool_call_count": 0,
"error_count": 0,
@ -84,6 +90,7 @@ def _ensure_run(session_id: Optional[str], task_id: Optional[str], platform: str
run = {
"run_id": rid,
"trace_id": tid,
"root_span_id": spans.new_span_id(),
"session_id": session_id or None,
"entrypoint": _entrypoint_for(platform),
"platform": platform or None,
@ -96,6 +103,33 @@ def _ensure_run(session_id: Optional[str], task_id: Optional[str], platform: str
return run
def _emit_call_span(run: Dict[str, Any], span_id: str, name: str, kind: str,
duration_ms: Optional[int], status: Optional[str]) -> None:
"""Emit the timing/lineage span for a model or tool call.
The call hooks fire on completion, so end_ns is ~now and start_ns is reconstructed
from the measured duration (end - duration). The span is parented to the run's root
so a 2-level run -> calls waterfall can be reconstructed from tel_spans.
"""
from agent.telemetry import emitter
from agent.telemetry.events import SpanEvent
end_ns = time.time_ns()
dur_ns = int(duration_ms) * 1_000_000 if isinstance(duration_ms, (int, float)) else 0
start_ns = end_ns - dur_ns
emitter.emit(SpanEvent(
span_id=span_id,
trace_id=run["trace_id"],
run_id=run["run_id"],
parent_span_id=run.get("root_span_id"),
name=name,
kind=kind,
start_ns=start_ns,
end_ns=end_ns,
status=status,
))
def _entrypoint_for(platform: Optional[str], source: Optional[str] = None) -> str:
"""Coarse entrypoint label (cli / gateway / tui / api / cron …).
@ -135,8 +169,9 @@ def _on_post_api_request(**kw: Any) -> None:
duration = kw.get("api_duration")
latency_ms = int(duration * 1000) if isinstance(duration, (int, float)) else None
span_id = spans.new_span_id()
evt = ModelCallEvent(
span_id=spans.new_span_id(),
span_id=span_id,
run_id=run["run_id"],
provider=kw.get("provider"), # raw
model=kw.get("model"), # raw
@ -151,6 +186,8 @@ def _on_post_api_request(**kw: Any) -> None:
)
with _runs_lock:
run["model_call_count"] += 1
_emit_call_span(run, span_id, name=kw.get("model") or "model_call",
kind="model", duration_ms=latency_ms, status="ok")
emitter.emit(evt)
@ -209,11 +246,15 @@ def _on_post_tool_call(**kw: Any) -> None:
if result_class == "error":
run["error_count"] += 1
dur_int = int(duration_ms) if isinstance(duration_ms, (int, float)) else None
span_id = spans.new_span_id()
_emit_call_span(run, span_id, name=function_name or "tool_call",
kind="tool", duration_ms=dur_int, status=result_class)
emitter.emit(ToolCallEvent(
span_id=spans.new_span_id(),
span_id=span_id,
run_id=run["run_id"],
tool_name=function_name, # raw tool name
duration_ms=int(duration_ms) if isinstance(duration_ms, (int, float)) else None,
duration_ms=dur_int,
result_class=result_class,
))
@ -245,7 +286,7 @@ def _tool_result_class(result: Any) -> str:
@_safe
def _on_session_finalize(**kw: Any) -> None:
from agent.telemetry import emitter, spans
from agent.telemetry.events import RunEvent
from agent.telemetry.events import RunEvent, SpanEvent
session_id = kw.get("session_id") or ""
key = _run_key(session_id, kw.get("task_id"))
@ -256,15 +297,29 @@ def _on_session_finalize(**kw: Any) -> None:
with _runs_lock:
_runs.pop(key, None)
end_ns = time.time_ns()
start_ns = run.get("start_ns", end_ns)
end_reason = _coarse_end_reason(kw)
# Root span for the run — the trace root the call spans hang off of.
emitter.emit(SpanEvent(
span_id=run.get("root_span_id") or spans.new_span_id(),
trace_id=run["trace_id"],
run_id=run["run_id"],
parent_span_id=None,
name=f"run:{run.get('entrypoint', 'cli')}",
kind="run",
start_ns=start_ns,
end_ns=end_ns,
status=end_reason,
))
emitter.emit(RunEvent(
run_id=run["run_id"],
trace_id=run["trace_id"],
entrypoint=run.get("entrypoint", "cli"),
session_id=run.get("session_id"),
platform=run.get("platform"),
start_ns=run.get("start_ns", time.time_ns()),
end_ns=time.time_ns(),
start_ns=start_ns,
end_ns=end_ns,
end_reason=end_reason,
model_call_count=run.get("model_call_count", 0),
tool_call_count=run.get("tool_call_count", 0),