mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-31 19:16:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

fix(monitoring): keep diagnostics content-free

Browse source
This commit is contained in:
Victor Kyriazakos 2026-07-22 23:34:21 +00:00
parent 18b4664543
commit 42bc6c8626
8 changed files with 29 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

1
agent/monitoring/events.py
View file

@ -50,7 +50,6 @@ class GatewayDiagnosticEvent:
subsystem: str
error_class: str = "unknown"
error_code: Optional[str] = None
redacted_message: Optional[str] = None
platform: Optional[str] = None
old_state: Optional[str] = None
new_state: Optional[str] = None

5
agent/monitoring/gateway_health.py
View file

@ -246,7 +246,6 @@ def build_gateway_health_snapshot(
platform=str(platform),
error_code=error_code,
error_class=classify_gateway_error(error_code or pdata.get("error_message")),
redacted_message=redact_gateway_message(pdata.get("error_message")),
profile=profile,
version=version,
severity="error" if state == "fatal" else "warning",
@ -325,7 +324,6 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current:
subsystem="gateway",
error_class=classify_gateway_error(current.get("exit_reason") or "startup_failed"),
error_code=classify_gateway_error(current.get("exit_reason") or "startup_failed"),
redacted_message=redact_gateway_message(current.get("exit_reason") or "startup failed"),
profile=profile,
version=version,
severity="error",
@ -374,7 +372,6 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current:
new_state=new_state,
error_code=error_code,
error_class=error_code,
redacted_message=redact_gateway_message(pdata.get("error_message")),
profile=profile,
version=version,
severity=severity,
@ -386,7 +383,6 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current:
platform=str(platform),
error_code=error_code,
error_class=error_code,
redacted_message=redact_gateway_message(pdata.get("error_message")),
profile=profile,
version=version,
severity=severity,
@ -426,7 +422,6 @@ class GatewayDiagnosticLogHandler(logging.Handler):
subsystem=subsystem,
platform=platform_for_subsystem(subsystem),
error_class=classify_gateway_error(message),
redacted_message=redact_gateway_message(message),
profile=self.profile,
version=self.version,
severity=record.levelname.lower(),

5
agent/monitoring/gateway_health_export.py
View file

@ -391,7 +391,10 @@ class GatewayDiagnosticLogStreamer:
if ev.get("event") != "gateway_diagnostic":
continue
attrs = _diagnostic_log_attributes(ev)
body = ev.get("redacted_message") or ev.get("name") or "gateway diagnostic"
# Rendered Python log messages may contain arbitrary user IDs, names,
# paths, or configured strings. Keep the OTLP body content-free and
# carry only the structured, allowlisted attributes above.
body = "gateway diagnostic"
record = self._LogRecord(
timestamp=ev.get("ts_ns"),
trace_id=self._sdk["INVALID_TRACE_ID"],

2
agent/monitoring/otlp_exporter.py
View file

@ -147,7 +147,7 @@ def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]:
"fatal_platform_count", "version",
"supervision_mode", "pid"),
"gateway_diagnostic": ("name", "subsystem", "error_class", "error_code",
"redacted_message", "platform", "old_state", "new_state",
"platform", "old_state", "new_state",
"version", "severity"),
}
for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type]

8
docs/observability/monitoring.md
View file

@ -1,11 +1,11 @@
# Gateway Monitoring
Service health monitoring plus redacted operational diagnostics for the
Service health monitoring plus structured operational diagnostics for the
Hermes gateway daemon, exported over OTLP/HTTP to an operator-configured
endpoint (OpenTelemetry Collector, DataDog, or any OTLP receiver).
This plane is content-free by construction. It exports gateway lifecycle
state, platform connector health, and redacted warning/error diagnostics.
state, platform connector health, and content-free warning/error diagnostics.
It never exports prompts, messages, tool arguments or results, session
history, usage analytics, audit logs, or execution traces. Run/model/tool
trajectory capture is a separate plane served by the NeMo Relay integration
@ -17,7 +17,7 @@ trajectory capture is a separate plane served by the NeMo Relay integration
| --- | --- | --- |
| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes |
| Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes |
| Diagnostics | `/v1/logs` | Warning/error gateway log events with secrets AND PII scrubbed in-process before egress (`[redacted]` / `[email]`), bounded error classes |
| Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported |
Signals carry `service.name`, version, supervision mode, and a stable one-way
hash of the install id so an operator can distinguish instances without
@ -85,7 +85,7 @@ python scripts/observability/otel_capture_collector.py \
--host 127.0.0.1 --port 4318 --log /tmp/hermes_otel_capture.jsonl
# terminal 2: drive the real exporter through lifecycle transitions,
# a fatal platform, and a redacted warning log, then flush
# a fatal platform, and a structured warning event, then flush
python scripts/observability/gateway_health_export_probe.py \
--endpoint http://127.0.0.1:4318/v1/traces \
--log /tmp/hermes_otel_capture.jsonl --wait 8

4
hermes_cli/main.py
View file

@ -14340,8 +14340,8 @@ def cmd_monitoring(args):
print(f" Diagnostic events: {'on' if gh.get('diagnostic_events_enabled', True) else 'off'}")
print(f" Warning/error logs: {'on' if gh.get('warning_error_events_enabled', True) else 'off'} "
f"(interval {gh.get('logs_export_interval_seconds', 5)}s)")
print(" Redaction: always on "
"(secrets/PII scrubbed in-process before egress; not configurable)")
print(" Content safety: always on "
"(rendered messages are never exported; not configurable)")
endpoint = otlp.get("endpoint") or ""
if otlp.get("enabled") and endpoint:
print(f" OTLP endpoint: {endpoint}")

20
tests/monitoring/test_gateway_health_export.py
View file

@ -111,8 +111,8 @@ def test_gateway_health_snapshot_emits_content_free_diagnostic_event():
assert health["fatal_platform_count"] == 1
assert platform["platform"] == "slack"
assert platform["error_code"] == "auth_failed"
assert "secret" not in platform["redacted_message"].lower()
assert "Bearer" not in platform["redacted_message"]
assert "redacted_message" not in platform
assert "Bearer" not in str(platform)
def test_gateway_health_snapshot_preserves_real_bounded_platform_states():
@ -147,7 +147,7 @@ def test_gateway_health_snapshot_preserves_real_bounded_platform_states():
assert observed == expected
def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog):
def test_gateway_diagnostic_log_handler_never_carries_rendered_message(caplog):
from agent.monitoring import emitter
from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
@ -166,7 +166,10 @@ def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog):
logger.addHandler(handler)
try:
logger.info("ignore info token sk-live-secret")
logger.warning("Slack token sk-live-secret failed for user@example.com")
logger.warning(
"Unauthorized user: acct_7f3a (Alice Smith) on slack; "
"token «redacted:sk-…»"
)
finally:
logger.removeHandler(handler)
finally:
@ -178,8 +181,9 @@ def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog):
assert event["name"] == "gateway.log.warning"
assert event["subsystem"] == "platform.slack"
assert event["error_class"] == "auth_failed"
assert "***" not in event["redacted_message"]
assert "user@example.com" not in event["redacted_message"]
assert "redacted_message" not in event
assert "acct_7f3a" not in str(event)
assert "Alice Smith" not in str(event)
def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypatch):
@ -223,7 +227,7 @@ def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypat
assert platform["old_state"] == "running"
assert platform["new_state"] == "fatal"
assert platform["error_code"] == "auth_failed"
assert "Bearer" not in platform["redacted_message"]
assert "redacted_message" not in platform
def test_runtime_status_transition_emits_startup_failed_and_exit():
@ -256,7 +260,7 @@ def test_runtime_status_transition_emits_startup_failed_and_exit():
assert "gateway.startup_failed" in names
assert "gateway.exit" in names
failed = next(e for e in captured if e["name"] == "gateway.startup_failed")
assert "***" not in failed["redacted_message"]
assert "redacted_message" not in failed
lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle")
assert lifecycle["exit_reason"] == "auth_failed"
exit_event = next(e for e in captured if e["name"] == "gateway.exit")

9
tests/monitoring/test_otlp_exporter.py
View file

@ -42,16 +42,19 @@ def test_gateway_health_event_maps_to_span_with_attrs():
assert attrs["hermes.active_agents"] == 2
def test_gateway_diagnostic_event_maps_redacted_message():
def test_gateway_diagnostic_event_drops_arbitrary_message_content():
provider, mem = _mem_provider()
OE.export_batch(provider, [{
"event": "gateway_diagnostic", "name": "platform.fatal",
"subsystem": "platform.slack", "error_class": "auth_failed",
"redacted_message": "token [redacted] rejected", "severity": "error",
"redacted_message": "Unauthorized user: acct_7f3a (Alice Smith)",
"severity": "error",
}])
attrs = dict(mem.get_finished_spans()[0].attributes or {})
assert attrs["hermes.error_class"] == "auth_failed"
assert attrs["hermes.redacted_message"] == "token [redacted] rejected"
assert "hermes.redacted_message" not in attrs
assert "acct_7f3a" not in str(attrs)
assert "Alice Smith" not in str(attrs)
def test_unknown_event_kind_exports_no_attrs_beyond_kind():