mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-12 13:52:15 +00:00
8a573bb6e7
Follow-up to the previous commit on #60432. The status-write guard (_consume_interrupted_flag, checked right before mark_job_run) closes the false-success bookkeeping gap, but run_one_job delivers its result BEFORE that check: delivery happens right after run_job() returns, mark_job_run happens at the very end. A job whose tool subprocess was killed mid-flight can still produce a plausible-looking final_response from the truncated output, and that response would reach the user via _deliver_result before the interrupted flag was ever consulted -- correct status in jobs.json, wrong message already sent. Adds _is_interrupted(), a non-destructive peek at the same _interrupted_job_ids set (_consume_interrupted_flag stays as the consuming, authoritative check right before the status write -- this needed a peek instead since the flag has to still be visible there). Checked right after save_job_output, before the deliver_content decision: if the run looked successful but was flagged interrupted, force success=False with an explicit interruption message. This routes delivery through the existing _summarize_cron_failure_for_delivery path (the same one a real failure already uses) instead of the raw final_response, so the user gets an honest "this run was interrupted" instead of a truncated/misleading result. Testing: 4 new tests in tests/cron/test_shutdown_interrupt.py -- _is_interrupted peek semantics (false/true/does-not-clear, as opposed to the consuming _consume_interrupted_flag), and the delivery-gate test itself, which mocks run_job to return a normal-looking success with a "plausible final response" while the job is pre-marked interrupted, and asserts _deliver_result receives the failure summary ("This run was interrupted.") instead, with the summarizer's error argument confirmed to mention the interruption. Fail-then-pass: reverted cron/scheduler.py only, the 4 new tests fail (3 on the missing _is_interrupted attribute, 1 -- the delivery-gate test -- on _summarize_cron_failure_for_delivery never being called, i.e. the raw response would have gone out); restored, all 16 tests in the file pass. Regression: tests/cron/ (683 tests) + test_cron_active_work_drain.py + test_gateway_shutdown.py + test_shutdown_cache_cleanup.py -- 11 pre-existing failures (Unix file-permission-bit and path-tilde assertions that don't apply on this Windows dev box), matching the same set already established as pre-existing in the prior commit's regression check. Zero new failures. Continues #60432
278 lines
11 KiB
Python
278 lines
11 KiB
Python
"""Tests for #60432: cron jobs must not be silently invisible to gateway
|
|
shutdown, and a job whose tool subprocess got killed by shutdown must
|
|
never be reported as a successful run.
|
|
|
|
Covers the cron/scheduler.py primitives directly:
|
|
- get_running_job_ids() -- thread-safe snapshot the gateway drain reads
|
|
- mark_running_jobs_interrupted() -- called by the gateway right after
|
|
it force-kills tool subprocesses
|
|
- the interrupted-flag race guard in run_one_job(), which must win over
|
|
the job's own thread finishing normally with a plausible-looking
|
|
result AFTER its tool was already killed out from under it
|
|
"""
|
|
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_scheduler_state():
|
|
"""Every test starts from a clean slate and leaves one behind, since
|
|
these sets are module-level globals shared across the test process."""
|
|
import cron.scheduler as sched
|
|
|
|
sched._running_job_ids.clear()
|
|
sched._interrupted_job_ids.clear()
|
|
yield
|
|
sched._running_job_ids.clear()
|
|
sched._interrupted_job_ids.clear()
|
|
|
|
|
|
class TestGetRunningJobIds:
|
|
def test_empty_when_nothing_running(self):
|
|
import cron.scheduler as sched
|
|
|
|
assert sched.get_running_job_ids() == frozenset()
|
|
|
|
def test_reflects_in_flight_jobs(self):
|
|
import cron.scheduler as sched
|
|
|
|
sched._running_job_ids.add("job-1")
|
|
sched._running_job_ids.add("job-2")
|
|
|
|
result = sched.get_running_job_ids()
|
|
|
|
assert result == frozenset({"job-1", "job-2"})
|
|
|
|
def test_snapshot_is_immutable_and_independent(self):
|
|
"""Mutating _running_job_ids after the call must not change the
|
|
already-returned snapshot -- callers (the gateway drain loop) rely
|
|
on this to safely count in a tight polling loop."""
|
|
import cron.scheduler as sched
|
|
|
|
sched._running_job_ids.add("job-1")
|
|
snapshot = sched.get_running_job_ids()
|
|
sched._running_job_ids.add("job-2")
|
|
|
|
assert snapshot == frozenset({"job-1"})
|
|
|
|
|
|
class TestMarkRunningJobsInterrupted:
|
|
def test_no_op_when_nothing_running(self):
|
|
import cron.scheduler as sched
|
|
|
|
with patch("cron.scheduler.mark_job_run") as mock_mark:
|
|
marked = sched.mark_running_jobs_interrupted("shutdown")
|
|
|
|
assert marked == []
|
|
mock_mark.assert_not_called()
|
|
|
|
def test_marks_every_in_flight_job(self):
|
|
import cron.scheduler as sched
|
|
|
|
sched._running_job_ids.update({"job-1", "job-2"})
|
|
|
|
with patch("cron.scheduler.mark_job_run") as mock_mark:
|
|
marked = sched.mark_running_jobs_interrupted("gateway shutdown (final-cleanup)")
|
|
|
|
assert sorted(marked) == ["job-1", "job-2"]
|
|
assert mock_mark.call_count == 2
|
|
called_ids = {c.args[0] for c in mock_mark.call_args_list}
|
|
assert called_ids == {"job-1", "job-2"}
|
|
for c in mock_mark.call_args_list:
|
|
# success must be False -- an interrupted run is never "ok".
|
|
assert c.args[1] is False
|
|
assert "gateway shutdown" in c.args[2]
|
|
|
|
def test_sets_interrupted_flag_for_consumption_by_run_one_job(self):
|
|
import cron.scheduler as sched
|
|
|
|
sched._running_job_ids.add("job-1")
|
|
|
|
with patch("cron.scheduler.mark_job_run"):
|
|
sched.mark_running_jobs_interrupted("shutdown")
|
|
|
|
assert "job-1" in sched._interrupted_job_ids
|
|
|
|
def test_one_job_marking_failure_does_not_block_the_others(self):
|
|
"""mark_job_run raising for one job (e.g. a jobs.json write race)
|
|
must not prevent the rest from being marked -- this runs during
|
|
shutdown, there's no retry window."""
|
|
import cron.scheduler as sched
|
|
|
|
sched._running_job_ids.update({"job-1", "job-2"})
|
|
|
|
def _side_effect(job_id, success, reason, **kwargs):
|
|
if job_id == "job-1":
|
|
raise OSError("disk full")
|
|
|
|
with patch("cron.scheduler.mark_job_run", side_effect=_side_effect):
|
|
marked = sched.mark_running_jobs_interrupted("shutdown")
|
|
|
|
assert marked == ["job-2"]
|
|
|
|
|
|
class TestIsInterrupted:
|
|
"""Peek-only check used at the delivery gate -- must NOT clear the
|
|
flag, unlike _consume_interrupted_flag."""
|
|
|
|
def test_false_when_not_marked(self):
|
|
import cron.scheduler as sched
|
|
|
|
assert sched._is_interrupted("job-1") is False
|
|
|
|
def test_true_when_marked(self):
|
|
import cron.scheduler as sched
|
|
|
|
sched._interrupted_job_ids.add("job-1")
|
|
|
|
assert sched._is_interrupted("job-1") is True
|
|
|
|
def test_does_not_clear_the_flag(self):
|
|
import cron.scheduler as sched
|
|
|
|
sched._interrupted_job_ids.add("job-1")
|
|
|
|
sched._is_interrupted("job-1")
|
|
|
|
# Still set -- the later, authoritative check before mark_job_run
|
|
# must still see it.
|
|
assert "job-1" in sched._interrupted_job_ids
|
|
assert sched._is_interrupted("job-1") is True
|
|
|
|
|
|
class TestConsumeInterruptedFlag:
|
|
def test_false_when_not_marked(self):
|
|
import cron.scheduler as sched
|
|
|
|
assert sched._consume_interrupted_flag("job-1") is False
|
|
|
|
def test_true_and_clears_when_marked(self):
|
|
import cron.scheduler as sched
|
|
|
|
sched._interrupted_job_ids.add("job-1")
|
|
|
|
assert sched._consume_interrupted_flag("job-1") is True
|
|
# Consumed -- a second check (e.g. a later, unrelated fire of the
|
|
# same recurring job ID) must not still read as interrupted.
|
|
assert sched._consume_interrupted_flag("job-1") is False
|
|
|
|
|
|
class TestRunOneJobHonoursInterruptedFlag:
|
|
"""run_one_job() must not let a job's own completion overwrite a
|
|
status the shutdown path already wrote for the same run."""
|
|
|
|
def _make_job(self, job_id="job-1"):
|
|
return {"id": job_id, "name": "test job", "prompt": "do work"}
|
|
|
|
def test_success_path_skipped_when_interrupted(self):
|
|
import cron.scheduler as sched
|
|
|
|
job = self._make_job()
|
|
sched._interrupted_job_ids.add(job["id"])
|
|
|
|
with patch("cron.scheduler.claim_dispatch", return_value=True), \
|
|
patch("agent.secret_scope.set_secret_scope", return_value=None), \
|
|
patch("agent.secret_scope.build_profile_secret_scope", return_value=None), \
|
|
patch("agent.secret_scope.reset_secret_scope"), \
|
|
patch(
|
|
"cron.scheduler.run_job",
|
|
return_value=(True, "full output", "final response", None),
|
|
), \
|
|
patch("cron.scheduler.save_job_output", return_value="/tmp/out.md"), \
|
|
patch("cron.scheduler._is_cron_silence_response", return_value=False), \
|
|
patch("cron.scheduler._deliver_result", return_value=None), \
|
|
patch("cron.scheduler.mark_job_run") as mock_mark:
|
|
result = sched.run_one_job(job)
|
|
|
|
assert result is True
|
|
# The would-be "success" write must NOT happen -- the shutdown
|
|
# path already wrote the authoritative interrupted status.
|
|
mock_mark.assert_not_called()
|
|
# Flag is consumed so a later, unrelated fire of the same job ID
|
|
# isn't permanently silenced.
|
|
assert job["id"] not in sched._interrupted_job_ids
|
|
|
|
def test_interrupted_job_delivers_failure_summary_not_raw_response(self):
|
|
"""The status-write guard alone isn't enough: delivery happens
|
|
BEFORE mark_job_run in run_one_job's own flow, so a job that kept
|
|
running post-kill and produced a plausible-looking final_response
|
|
must not have that response sent to the user just because the
|
|
eventual status write gets suppressed. Interrupted jobs must route
|
|
through the same failure-summary delivery path a real failure
|
|
would."""
|
|
import cron.scheduler as sched
|
|
|
|
job = self._make_job()
|
|
sched._interrupted_job_ids.add(job["id"])
|
|
|
|
with patch("cron.scheduler.claim_dispatch", return_value=True), \
|
|
patch("agent.secret_scope.set_secret_scope", return_value=None), \
|
|
patch("agent.secret_scope.build_profile_secret_scope", return_value=None), \
|
|
patch("agent.secret_scope.reset_secret_scope"), \
|
|
patch(
|
|
"cron.scheduler.run_job",
|
|
return_value=(True, "full output", "a plausible final response", None),
|
|
), \
|
|
patch("cron.scheduler.save_job_output", return_value="/tmp/out.md"), \
|
|
patch(
|
|
"cron.scheduler._summarize_cron_failure_for_delivery",
|
|
return_value="This run was interrupted.",
|
|
) as mock_summarize, \
|
|
patch("cron.scheduler._is_cron_silence_response", return_value=False), \
|
|
patch("cron.scheduler._deliver_result", return_value=None) as mock_deliver, \
|
|
patch("cron.scheduler.mark_job_run"):
|
|
result = sched.run_one_job(job)
|
|
|
|
assert result is True
|
|
mock_summarize.assert_called_once()
|
|
# The summarizer's error argument must mention the interruption,
|
|
# not be silently None / the agent's own (possibly absent) error.
|
|
assert "interrupt" in mock_summarize.call_args.args[1].lower()
|
|
delivered_content = mock_deliver.call_args.args[1]
|
|
assert delivered_content == "This run was interrupted."
|
|
assert "plausible final response" not in delivered_content
|
|
|
|
def test_success_path_writes_normally_when_not_interrupted(self):
|
|
"""Control case: the guard must not swallow ordinary, un-interrupted
|
|
completions -- only ones the shutdown path explicitly flagged."""
|
|
import cron.scheduler as sched
|
|
|
|
job = self._make_job()
|
|
|
|
with patch("cron.scheduler.claim_dispatch", return_value=True), \
|
|
patch("agent.secret_scope.set_secret_scope", return_value=None), \
|
|
patch("agent.secret_scope.build_profile_secret_scope", return_value=None), \
|
|
patch("agent.secret_scope.reset_secret_scope"), \
|
|
patch(
|
|
"cron.scheduler.run_job",
|
|
return_value=(True, "full output", "final response", None),
|
|
), \
|
|
patch("cron.scheduler.save_job_output", return_value="/tmp/out.md"), \
|
|
patch("cron.scheduler._is_cron_silence_response", return_value=False), \
|
|
patch("cron.scheduler._deliver_result", return_value=None), \
|
|
patch("cron.scheduler.mark_job_run") as mock_mark:
|
|
result = sched.run_one_job(job)
|
|
|
|
assert result is True
|
|
mock_mark.assert_called_once()
|
|
assert mock_mark.call_args.args[0] == job["id"]
|
|
assert mock_mark.call_args.args[1] is True # success
|
|
|
|
def test_exception_path_also_honours_interrupted_flag(self):
|
|
import cron.scheduler as sched
|
|
|
|
job = self._make_job()
|
|
sched._interrupted_job_ids.add(job["id"])
|
|
|
|
with patch("cron.scheduler.claim_dispatch", return_value=True), \
|
|
patch("agent.secret_scope.set_secret_scope", return_value=None), \
|
|
patch("agent.secret_scope.build_profile_secret_scope", return_value=None), \
|
|
patch("agent.secret_scope.reset_secret_scope"), \
|
|
patch("cron.scheduler.run_job", side_effect=RuntimeError("boom")), \
|
|
patch("cron.scheduler.mark_job_run") as mock_mark:
|
|
result = sched.run_one_job(job)
|
|
|
|
assert result is False
|
|
mock_mark.assert_not_called()
|