mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-12 13:52:15 +00:00
5986cdd380
Defense-in-depth alongside the interpreter-shutdown guard: run_job closed the cron agent's async resources (agent.close + cleanup_stale_async_clients) in its finally block BEFORE run_one_job called _deliver_result, so a live delivery could race a torn-down async client. run_job now accepts an optional defer_agent_teardown holder; when set it hands the live agent back instead of closing it, and run_one_job tears it down (via the extracted _teardown_cron_agent helper) only AFTER delivery — in a finally so a failed run never leaks. Default path (holder=None) is unchanged, so every existing caller keeps inline teardown. Reorder approach based on #58777 by @LavyaTandel; reworked to keep a single delivery site in run_one_job and add regression coverage. Co-authored-by: LavyaTandel <lavya@loom.local>
276 lines
11 KiB
Python
276 lines
11 KiB
Python
"""Characterization + unit tests for the `run_one_job` shared helper (Phase 4A).
|
|
|
|
`tick`'s per-job body (`_process_job`) is the execute → save → deliver → mark
|
|
sequence that fires ONE due job. Phase 4A extracts it into a module-level
|
|
`run_one_job(job, *, adapters=None, loop=None, verbose=False)` so the external
|
|
Chronos provider's `fire_due` can reuse the IDENTICAL body — no duplicated
|
|
correctness.
|
|
|
|
The first test characterizes the sequence as driven through `tick()` (proving
|
|
the extraction didn't change `tick`'s behavior); the rest unit-test the
|
|
extracted helper directly.
|
|
"""
|
|
import cron.scheduler as s
|
|
|
|
|
|
def _patch_pipeline(monkeypatch, *, success=True, output="out", final="final response",
|
|
error=None, silent_marker_in=None):
|
|
"""Patch the job pipeline primitives and record the call order."""
|
|
calls = []
|
|
|
|
def fake_run_job(job, *, defer_agent_teardown=None):
|
|
calls.append(("run_job", job["id"]))
|
|
fr = final if silent_marker_in is None else silent_marker_in
|
|
return (success, output, fr, error)
|
|
|
|
def fake_save(jid, out):
|
|
calls.append(("save", jid))
|
|
return f"/tmp/{jid}.txt"
|
|
|
|
def fake_deliver(job, content, adapters=None, loop=None):
|
|
calls.append(("deliver", job["id"]))
|
|
return None
|
|
|
|
def fake_mark(jid, ok, err=None, delivery_error=None):
|
|
calls.append(("mark", jid, ok))
|
|
|
|
monkeypatch.setattr(s, "run_job", fake_run_job)
|
|
monkeypatch.setattr(s, "save_job_output", fake_save)
|
|
monkeypatch.setattr(s, "_deliver_result", fake_deliver)
|
|
monkeypatch.setattr(s, "mark_job_run", fake_mark)
|
|
return calls
|
|
|
|
|
|
def test_tick_process_job_sequence(monkeypatch):
|
|
"""Characterization: a single due job driven through tick() runs the
|
|
sequence run_job → save → deliver → mark, in that order."""
|
|
calls = _patch_pipeline(monkeypatch)
|
|
monkeypatch.setattr(s, "get_due_jobs", lambda: [{"id": "j1", "name": "t"}])
|
|
monkeypatch.setattr(s, "advance_next_run", lambda jid: True)
|
|
|
|
s.tick(verbose=False, sync=True)
|
|
|
|
assert [c[0] for c in calls] == ["run_job", "save", "deliver", "mark"]
|
|
assert calls[-1] == ("mark", "j1", True)
|
|
|
|
|
|
def test_run_one_job_success_sequence(monkeypatch):
|
|
"""The extracted helper runs the same execute→save→deliver→mark sequence
|
|
for a successful job."""
|
|
calls = _patch_pipeline(monkeypatch)
|
|
|
|
ok = s.run_one_job({"id": "j2", "name": "t"})
|
|
|
|
assert ok is True
|
|
assert [c[0] for c in calls] == ["run_job", "save", "deliver", "mark"]
|
|
assert calls[-1] == ("mark", "j2", True)
|
|
|
|
|
|
def test_run_one_job_silent_skips_delivery(monkeypatch):
|
|
"""A [SILENT] final response saves output + marks the run but does NOT
|
|
deliver."""
|
|
calls = _patch_pipeline(monkeypatch, silent_marker_in="[SILENT]")
|
|
|
|
s.run_one_job({"id": "j3", "name": "t"})
|
|
|
|
kinds = [c[0] for c in calls]
|
|
assert "run_job" in kinds and "save" in kinds and "mark" in kinds
|
|
assert "deliver" not in kinds
|
|
|
|
|
|
def test_run_one_job_empty_response_is_soft_failure(monkeypatch):
|
|
"""An empty final response marks the run as NOT ok (issue #8585)."""
|
|
calls = _patch_pipeline(monkeypatch, final=" ")
|
|
|
|
s.run_one_job({"id": "j4", "name": "t"})
|
|
|
|
mark = [c for c in calls if c[0] == "mark"][0]
|
|
assert mark == ("mark", "j4", False)
|
|
|
|
|
|
def test_run_one_job_failed_job_delivers_error(monkeypatch):
|
|
"""A failed job still delivers (the error notice) and marks not-ok."""
|
|
calls = _patch_pipeline(monkeypatch, success=False, final="", error="boom")
|
|
|
|
s.run_one_job({"id": "j5", "name": "t"})
|
|
|
|
kinds = [c[0] for c in calls]
|
|
assert "deliver" in kinds # failures always deliver
|
|
mark = [c for c in calls if c[0] == "mark"][0]
|
|
assert mark == ("mark", "j5", False)
|
|
|
|
|
|
def test_run_one_job_exception_marks_failure(monkeypatch):
|
|
"""If run_job raises, the helper marks the run failed and returns False
|
|
rather than propagating."""
|
|
def boom(job, *, defer_agent_teardown=None):
|
|
raise RuntimeError("kaboom")
|
|
|
|
monkeypatch.setattr(s, "run_job", boom)
|
|
marks = []
|
|
monkeypatch.setattr(
|
|
s, "mark_job_run",
|
|
lambda jid, ok, err=None, delivery_error=None: marks.append((jid, ok)),
|
|
)
|
|
|
|
ok = s.run_one_job({"id": "j6", "name": "t"})
|
|
|
|
assert ok is False
|
|
assert marks == [("j6", False)]
|
|
|
|
|
|
def test_run_one_job_installs_secret_scope_under_multiplex(monkeypatch, tmp_path):
|
|
"""Regression: under profile isolation (multiplex active), run_one_job must
|
|
execute run_job inside a profile secret scope so credential reads
|
|
(resolve_runtime_provider -> get_secret) don't fail-close with
|
|
UnscopedSecretError, and must tear the scope down afterward.
|
|
|
|
Behavior contract: a scope is present during run_job and absent after,
|
|
regardless of the concrete secret values.
|
|
"""
|
|
from agent import secret_scope as ss
|
|
|
|
# Point cron's home resolution at a profile whose .env carries a secret.
|
|
(tmp_path / ".env").write_text("OPENROUTER_BASE_URL=https://openrouter.ai/api/v1\n")
|
|
monkeypatch.setattr(s, "_get_hermes_home", lambda: tmp_path)
|
|
|
|
scope_during_run = {}
|
|
|
|
def fake_run_job(job, *, defer_agent_teardown=None):
|
|
# This is where resolve_runtime_provider() would read a secret. Prove a
|
|
# scope is installed and the profile's secret resolves without raising.
|
|
scope_during_run["scope"] = ss.current_secret_scope()
|
|
scope_during_run["base_url"] = ss.get_secret("OPENROUTER_BASE_URL")
|
|
return (True, "out", "final", None)
|
|
|
|
monkeypatch.setattr(s, "run_job", fake_run_job)
|
|
monkeypatch.setattr(s, "save_job_output", lambda jid, out: f"/tmp/{jid}.txt")
|
|
monkeypatch.setattr(s, "_deliver_result", lambda *a, **k: None)
|
|
monkeypatch.setattr(s, "mark_job_run", lambda *a, **k: None)
|
|
|
|
ss.set_multiplex_active(True)
|
|
try:
|
|
ok = s.run_one_job({"id": "j7", "name": "t"})
|
|
finally:
|
|
ss.set_multiplex_active(False)
|
|
|
|
assert ok is True
|
|
# Scope was installed during run_job and the profile secret resolved.
|
|
assert scope_during_run["scope"] is not None
|
|
assert scope_during_run["base_url"] == "https://openrouter.ai/api/v1"
|
|
# And it was torn down after run_one_job returned (no leak).
|
|
assert ss.current_secret_scope() is None
|
|
|
|
|
|
def test_run_one_job_delivers_before_agent_teardown(monkeypatch):
|
|
"""Regression for #58720: the cron agent's async-resource teardown
|
|
(agent.close + cleanup_stale_async_clients) MUST run AFTER delivery, not
|
|
before. run_job defers teardown by appending the live agent to the holder
|
|
list; run_one_job tears it down only after _deliver_result has run. If the
|
|
order flips, delivery races a torn-down async client and dies with
|
|
'cannot schedule new futures after interpreter shutdown'.
|
|
"""
|
|
order = []
|
|
|
|
class FakeAgent:
|
|
def close(self):
|
|
order.append("agent.close")
|
|
|
|
def fake_run_job(job, *, defer_agent_teardown=None):
|
|
order.append("run_job")
|
|
# Mimic run_job's deferral contract: hand the live agent back so the
|
|
# caller tears it down after delivery instead of in run_job's finally.
|
|
assert defer_agent_teardown is not None, "run_one_job must defer teardown"
|
|
defer_agent_teardown.append(FakeAgent())
|
|
return (True, "out", "final response", None)
|
|
|
|
def fake_deliver(job, content, adapters=None, loop=None):
|
|
order.append("deliver")
|
|
return None
|
|
|
|
monkeypatch.setattr(s, "run_job", fake_run_job)
|
|
monkeypatch.setattr(s, "save_job_output", lambda jid, out: f"/tmp/{jid}.txt")
|
|
monkeypatch.setattr(s, "_deliver_result", fake_deliver)
|
|
monkeypatch.setattr(s, "mark_job_run", lambda *a, **k: None)
|
|
# cleanup_stale_async_clients is imported lazily inside _teardown_cron_agent;
|
|
# stub it so the teardown records its own marker without touching real caches.
|
|
import agent.auxiliary_client as aux
|
|
monkeypatch.setattr(aux, "cleanup_stale_async_clients",
|
|
lambda: order.append("cleanup_stale"))
|
|
|
|
ok = s.run_one_job({"id": "j8", "name": "t"})
|
|
|
|
assert ok is True
|
|
# Delivery must strictly precede agent teardown + stale-client reap.
|
|
assert order == ["run_job", "deliver", "agent.close", "cleanup_stale"], order
|
|
|
|
|
|
def test_run_one_job_tears_down_deferred_agent_when_delivery_raises(monkeypatch):
|
|
"""Even if _deliver_result raises, the deferred agent is still torn down
|
|
(no fd/client leak — #10200). Teardown lives in a finally around delivery.
|
|
"""
|
|
order = []
|
|
|
|
class FakeAgent:
|
|
def close(self):
|
|
order.append("agent.close")
|
|
|
|
def fake_run_job(job, *, defer_agent_teardown=None):
|
|
defer_agent_teardown.append(FakeAgent())
|
|
return (True, "out", "final response", None)
|
|
|
|
def boom_deliver(job, content, adapters=None, loop=None):
|
|
order.append("deliver-raise")
|
|
raise RuntimeError("send blew up")
|
|
|
|
monkeypatch.setattr(s, "run_job", fake_run_job)
|
|
monkeypatch.setattr(s, "save_job_output", lambda jid, out: f"/tmp/{jid}.txt")
|
|
monkeypatch.setattr(s, "_deliver_result", boom_deliver)
|
|
monkeypatch.setattr(s, "mark_job_run", lambda *a, **k: None)
|
|
import agent.auxiliary_client as aux
|
|
monkeypatch.setattr(aux, "cleanup_stale_async_clients",
|
|
lambda: order.append("cleanup_stale"))
|
|
|
|
ok = s.run_one_job({"id": "j9", "name": "t"})
|
|
|
|
assert ok is True # delivery error is recorded, not propagated
|
|
assert order == ["deliver-raise", "agent.close", "cleanup_stale"], order
|
|
|
|
|
|
def test_run_one_job_tears_down_deferred_agent_when_save_raises(monkeypatch):
|
|
"""#58720 W1: if save_job_output (or the [SILENT]/empty computation) raises
|
|
AFTER run_job hands the agent back but BEFORE delivery, the deferred agent
|
|
must still be torn down. The outer `except` would otherwise swallow the
|
|
error and leak the agent (#10200). Teardown lives in a finally spanning
|
|
save→deliver.
|
|
"""
|
|
order = []
|
|
|
|
class FakeAgent:
|
|
def close(self):
|
|
order.append("agent.close")
|
|
|
|
def fake_run_job(job, *, defer_agent_teardown=None):
|
|
defer_agent_teardown.append(FakeAgent())
|
|
return (True, "out", "final response", None)
|
|
|
|
def boom_save(jid, out):
|
|
order.append("save-raise")
|
|
raise RuntimeError("disk full")
|
|
|
|
monkeypatch.setattr(s, "run_job", fake_run_job)
|
|
monkeypatch.setattr(s, "save_job_output", boom_save)
|
|
monkeypatch.setattr(s, "_deliver_result",
|
|
lambda *a, **k: order.append("deliver"))
|
|
monkeypatch.setattr(s, "mark_job_run", lambda *a, **k: None)
|
|
import agent.auxiliary_client as aux
|
|
monkeypatch.setattr(aux, "cleanup_stale_async_clients",
|
|
lambda: order.append("cleanup_stale"))
|
|
|
|
ok = s.run_one_job({"id": "j10", "name": "t"})
|
|
|
|
# save raised → outer handler marks failure and returns False, but the
|
|
# deferred agent was still torn down (no delivery, no leak).
|
|
assert ok is False
|
|
assert "deliver" not in order
|
|
assert order == ["save-raise", "agent.close", "cleanup_stale"], order
|