diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index 422650a5dcd..0a377c99ab3 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -334,6 +334,27 @@ def _read_background_work_count() -> int: return total +def _read_background_delegations_count() -> int: + """Count live async delegation UNITS (dispatch/pool slots). + + Complements ``_read_background_work_count`` (which is task-granular): this + counts each ``delegate_task`` dispatch as ONE regardless of fan-out width, + matching the async pool's capacity accounting (a batch = one slot). Together + the two metrics let an operator see both slot pressure + (``background_delegations``, alert vs ``max_concurrent_children``) and real + concurrent subagent load (``background_work``). Delegations only — it does + not include ``terminal(background)`` / kanban work, which are already folded + into ``background_work``. Best-effort; 0 if the source can't be imported. + """ + try: + from tools.async_delegation import active_count + + return max(0, int(active_count())) + except Exception: + logger.debug("background-delegations count failed", exc_info=True) + return 0 + + def _read_runtime_snapshot(config: Dict[str, Any]): gateway_snapshot = _read_gateway_snapshot(config) # Background/subagent work — a distinct metric from active_agents (which @@ -350,6 +371,13 @@ def _read_runtime_snapshot(config: Dict[str, Any]): attributes=base, ) ) + gateway_snapshot.metrics.append( + GatewayMetric( + name="hermes.gateway.background_delegations", + value=_read_background_delegations_count(), + attributes=base, + ) + ) except Exception as exc: logger.warning( "background-work snapshot unavailable; metric not exported (error_type=%s)", @@ -414,6 +442,7 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: "hermes.gateway.drainable", "hermes.gateway.restart_requested", "hermes.gateway.background_work", + "hermes.gateway.background_delegations", "hermes.platform.up", "hermes.platform.degraded", "hermes.cron.scheduler.heartbeat_age_seconds", diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index 742497a6f33..f798bf2dbe6 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -16,7 +16,7 @@ capture is a separate plane served by the NeMo Relay integration | Signal | OTLP route | Content | | --- | --- | --- | -| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/background_work/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | +| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/background_work/background_delegations/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | | Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | | Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported | | Cron scheduler gauges | `/v1/metrics` | Ticker heartbeat and last-success age (omitted when unavailable), a monotonic catch-up-occurrence count from the scheduler's stale-window branch, enabled/running job counts, and overdue count derived from persisted `next_run_at` plus the scheduler's existing grace rule | @@ -26,16 +26,19 @@ Signals carry `service.name`, version, supervision mode, and a stable one-way hash of the install id so an operator can distinguish instances without exporting account/profile identity or the raw install identifier. -`hermes.gateway.active_agents` and `hermes.gateway.background_work` are -complementary and deliberately disjoint. `active_agents` counts foreground -message turns plus in-flight cron jobs plus API runs — the work the gateway -drains on shutdown. `background_work` counts detached work that `active_agents` -never includes: backgrounded `delegate_task` subagents, `terminal(background=true)` -processes, and kanban workers. `background_work` is **task-granular** — a -fan-out batch of N subagents counts as N, not as one dispatch unit — so it -reflects real concurrent subagent load rather than async-pool slot usage (a -batch occupies one slot but runs N children). Sum both for total live work per -instance. +`hermes.gateway.active_agents`, `hermes.gateway.background_work`, and +`hermes.gateway.background_delegations` are complementary. `active_agents` +counts foreground message turns plus in-flight cron jobs plus API runs — the +work the gateway drains on shutdown. `background_work` counts detached work that +`active_agents` never includes: backgrounded `delegate_task` subagents, +`terminal(background=true)` processes, and kanban workers; it is +**task-granular** — a fan-out batch of N subagents counts as N — so it reflects +real concurrent subagent load. `background_delegations` counts only async +delegation **units** (each `delegate_task` dispatch is one, a fan-out batch is +one), matching the async pool's capacity accounting; alert it against +`delegation.max_concurrent_children` to see slot pressure. Sum `active_agents` +and `background_work` for total live work per instance; use +`background_delegations` for pool-saturation. ## Enabling diff --git a/tests/monitoring/test_cron_health_export.py b/tests/monitoring/test_cron_health_export.py index 2b78b020e11..adc0e91c303 100644 --- a/tests/monitoring/test_cron_health_export.py +++ b/tests/monitoring/test_cron_health_export.py @@ -193,14 +193,41 @@ def test_gateway_export_includes_cron_metrics_and_only_accepted_event_planes(mon names = [metric.name for metric in snapshot.metrics] # Cron metrics are folded into the gateway snapshot... assert "hermes.cron.jobs.enabled" in names - # ...and the background/subagent-work gauge is appended (distinct from + # ...and the background/subagent-work gauges are appended (distinct from # active_agents). Assert the relationship, not a frozen exact list. assert "hermes.gateway.background_work" in names + assert "hermes.gateway.background_delegations" in names assert gateway_health_export._gateway_health_event({"event": "cron_execution"}) is True assert gateway_health_export._gateway_health_event({"event": "gateway_health"}) is True assert gateway_health_export._gateway_health_event({"event": "run"}) is False +def test_background_work_is_task_granular_and_delegations_is_unit_granular(monkeypatch): + """background_work expands batches to child tasks; background_delegations + counts dispatch units. A 3-task batch => work +3, delegations +1. + """ + from agent.monitoring import gateway_health_export + from tools import async_delegation as ad + + with ad._records_lock: + saved = dict(ad._records) + ad._records.clear() + ad._records["single"] = {"status": "running"} + ad._records["batch3"] = {"status": "running", "is_batch": True, "goals": ["a", "b", "c"]} + # Isolate from process_registry so we measure only the delegation contribution. + monkeypatch.setattr( + "tools.process_registry.process_registry.count_running", lambda: 0, raising=False + ) + try: + # work = single(1) + batch(3) = 4 tasks; delegations = 2 units. + assert gateway_health_export._read_background_work_count() == 4 + assert gateway_health_export._read_background_delegations_count() == 2 + finally: + with ad._records_lock: + ad._records.clear() + ad._records.update(saved) + + def test_registered_observable_metric_names_cover_snapshot_metrics(monkeypatch): """Every gauge emitted in the runtime snapshot must also be registered in the observable-gauge metric_names list, or the OTLP exporter never observes it.