mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
feat(monitoring): add hermes.gateway.background_delegations (unit/slot count)
Complements the task-granular background_work with the async-delegation UNIT count (each dispatch/batch = 1), recovering the pool-slot semantics active_count() gives. Together: background_work = real concurrent subagent load (batch expanded), background_delegations = slot pressure to alert against delegation.max_concurrent_children. Registered in metric_names, documented, and covered by a task-vs-unit contract test.
This commit is contained in:
Victor Kyriazakos
parent
0ef0816284
commit
cc80dba96f
3 changed files with 71 additions and 12 deletions
|
|
@ -334,6 +334,27 @@ def _read_background_work_count() -> int:
|
|||
return total
|
||||
|
||||
|
||||
def _read_background_delegations_count() -> int:
|
||||
"""Count live async delegation UNITS (dispatch/pool slots).
|
||||
|
||||
Complements ``_read_background_work_count`` (which is task-granular): this
|
||||
counts each ``delegate_task`` dispatch as ONE regardless of fan-out width,
|
||||
matching the async pool's capacity accounting (a batch = one slot). Together
|
||||
the two metrics let an operator see both slot pressure
|
||||
(``background_delegations``, alert vs ``max_concurrent_children``) and real
|
||||
concurrent subagent load (``background_work``). Delegations only — it does
|
||||
not include ``terminal(background)`` / kanban work, which are already folded
|
||||
into ``background_work``. Best-effort; 0 if the source can't be imported.
|
||||
"""
|
||||
try:
|
||||
from tools.async_delegation import active_count
|
||||
|
||||
return max(0, int(active_count()))
|
||||
except Exception:
|
||||
logger.debug("background-delegations count failed", exc_info=True)
|
||||
return 0
|
||||
|
||||
|
||||
def _read_runtime_snapshot(config: Dict[str, Any]):
|
||||
gateway_snapshot = _read_gateway_snapshot(config)
|
||||
# Background/subagent work — a distinct metric from active_agents (which
|
||||
|
|
@ -350,6 +371,13 @@ def _read_runtime_snapshot(config: Dict[str, Any]):
|
|||
attributes=base,
|
||||
)
|
||||
)
|
||||
gateway_snapshot.metrics.append(
|
||||
GatewayMetric(
|
||||
name="hermes.gateway.background_delegations",
|
||||
value=_read_background_delegations_count(),
|
||||
attributes=base,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"background-work snapshot unavailable; metric not exported (error_type=%s)",
|
||||
|
|
@ -414,6 +442,7 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any:
|
|||
"hermes.gateway.drainable",
|
||||
"hermes.gateway.restart_requested",
|
||||
"hermes.gateway.background_work",
|
||||
"hermes.gateway.background_delegations",
|
||||
"hermes.platform.up",
|
||||
"hermes.platform.degraded",
|
||||
"hermes.cron.scheduler.heartbeat_age_seconds",
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ capture is a separate plane served by the NeMo Relay integration
|
|||
|
||||
| Signal | OTLP route | Content |
|
||||
| --- | --- | --- |
|
||||
| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/background_work/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes |
|
||||
| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/background_work/background_delegations/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes |
|
||||
| Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes |
|
||||
| Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported |
|
||||
| Cron scheduler gauges | `/v1/metrics` | Ticker heartbeat and last-success age (omitted when unavailable), a monotonic catch-up-occurrence count from the scheduler's stale-window branch, enabled/running job counts, and overdue count derived from persisted `next_run_at` plus the scheduler's existing grace rule |
|
||||
|
|
@ -26,16 +26,19 @@ Signals carry `service.name`, version, supervision mode, and a stable one-way
|
|||
hash of the install id so an operator can distinguish instances without
|
||||
exporting account/profile identity or the raw install identifier.
|
||||
|
||||
`hermes.gateway.active_agents` and `hermes.gateway.background_work` are
|
||||
complementary and deliberately disjoint. `active_agents` counts foreground
|
||||
message turns plus in-flight cron jobs plus API runs — the work the gateway
|
||||
drains on shutdown. `background_work` counts detached work that `active_agents`
|
||||
never includes: backgrounded `delegate_task` subagents, `terminal(background=true)`
|
||||
processes, and kanban workers. `background_work` is **task-granular** — a
|
||||
fan-out batch of N subagents counts as N, not as one dispatch unit — so it
|
||||
reflects real concurrent subagent load rather than async-pool slot usage (a
|
||||
batch occupies one slot but runs N children). Sum both for total live work per
|
||||
instance.
|
||||
`hermes.gateway.active_agents`, `hermes.gateway.background_work`, and
|
||||
`hermes.gateway.background_delegations` are complementary. `active_agents`
|
||||
counts foreground message turns plus in-flight cron jobs plus API runs — the
|
||||
work the gateway drains on shutdown. `background_work` counts detached work that
|
||||
`active_agents` never includes: backgrounded `delegate_task` subagents,
|
||||
`terminal(background=true)` processes, and kanban workers; it is
|
||||
**task-granular** — a fan-out batch of N subagents counts as N — so it reflects
|
||||
real concurrent subagent load. `background_delegations` counts only async
|
||||
delegation **units** (each `delegate_task` dispatch is one, a fan-out batch is
|
||||
one), matching the async pool's capacity accounting; alert it against
|
||||
`delegation.max_concurrent_children` to see slot pressure. Sum `active_agents`
|
||||
and `background_work` for total live work per instance; use
|
||||
`background_delegations` for pool-saturation.
|
||||
|
||||
## Enabling
|
||||
|
||||
|
|
|
|||
|
|
@ -193,14 +193,41 @@ def test_gateway_export_includes_cron_metrics_and_only_accepted_event_planes(mon
|
|||
names = [metric.name for metric in snapshot.metrics]
|
||||
# Cron metrics are folded into the gateway snapshot...
|
||||
assert "hermes.cron.jobs.enabled" in names
|
||||
# ...and the background/subagent-work gauge is appended (distinct from
|
||||
# ...and the background/subagent-work gauges are appended (distinct from
|
||||
# active_agents). Assert the relationship, not a frozen exact list.
|
||||
assert "hermes.gateway.background_work" in names
|
||||
assert "hermes.gateway.background_delegations" in names
|
||||
assert gateway_health_export._gateway_health_event({"event": "cron_execution"}) is True
|
||||
assert gateway_health_export._gateway_health_event({"event": "gateway_health"}) is True
|
||||
assert gateway_health_export._gateway_health_event({"event": "run"}) is False
|
||||
|
||||
|
||||
def test_background_work_is_task_granular_and_delegations_is_unit_granular(monkeypatch):
|
||||
"""background_work expands batches to child tasks; background_delegations
|
||||
counts dispatch units. A 3-task batch => work +3, delegations +1.
|
||||
"""
|
||||
from agent.monitoring import gateway_health_export
|
||||
from tools import async_delegation as ad
|
||||
|
||||
with ad._records_lock:
|
||||
saved = dict(ad._records)
|
||||
ad._records.clear()
|
||||
ad._records["single"] = {"status": "running"}
|
||||
ad._records["batch3"] = {"status": "running", "is_batch": True, "goals": ["a", "b", "c"]}
|
||||
# Isolate from process_registry so we measure only the delegation contribution.
|
||||
monkeypatch.setattr(
|
||||
"tools.process_registry.process_registry.count_running", lambda: 0, raising=False
|
||||
)
|
||||
try:
|
||||
# work = single(1) + batch(3) = 4 tasks; delegations = 2 units.
|
||||
assert gateway_health_export._read_background_work_count() == 4
|
||||
assert gateway_health_export._read_background_delegations_count() == 2
|
||||
finally:
|
||||
with ad._records_lock:
|
||||
ad._records.clear()
|
||||
ad._records.update(saved)
|
||||
|
||||
|
||||
def test_registered_observable_metric_names_cover_snapshot_metrics(monkeypatch):
|
||||
"""Every gauge emitted in the runtime snapshot must also be registered in the
|
||||
observable-gauge metric_names list, or the OTLP exporter never observes it.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue