"""Tests for async (background) delegation — tools/async_delegation.py. Covers the dispatch handle, non-blocking behavior, completion-event delivery onto the shared process_registry.completion_queue, the rich re-injection block formatting, capacity rejection, and crash handling. """ import json import os import queue import subprocess import sys import threading import time import pytest from tools import async_delegation as ad from tools.process_registry import process_registry, format_process_notification @pytest.fixture(autouse=True) def _clean_state(): ad._reset_for_tests() while not process_registry.completion_queue.empty(): process_registry.completion_queue.get_nowait() yield # Give just-released workers a beat to finalize BEFORE draining, so their # completion events land now instead of leaking into the next test's # queue (worker threads push events asynchronously; a drain that races an # in-flight _finalize misses it). deadline = time.monotonic() + 2.0 while ad.active_count() and time.monotonic() < deadline: time.sleep(0.02) ad._reset_for_tests() while not process_registry.completion_queue.empty(): process_registry.completion_queue.get_nowait() def _drain_one(timeout=5.0): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if not process_registry.completion_queue.empty(): return process_registry.completion_queue.get_nowait() time.sleep(0.02) return None def _drain_for(delegation_id, timeout=5.0): """Drain until the event for *delegation_id* appears (discarding others). Completion events are pushed asynchronously by worker threads, so a straggler from a PREVIOUS test can land after that test's teardown drain and leak into the current test's queue. Matching on delegation_id makes the assertion immune to that cross-test leak. """ deadline = time.monotonic() + timeout while time.monotonic() < deadline: if not process_registry.completion_queue.empty(): evt = process_registry.completion_queue.get_nowait() if evt.get("delegation_id") == delegation_id: return evt continue time.sleep(0.02) return None def test_dispatch_returns_immediately_without_blocking(): gate = threading.Event() def runner(): gate.wait(timeout=60) return {"status": "completed", "summary": "done", "api_calls": 1, "duration_seconds": 0.1, "model": "m"} t0 = time.monotonic() res = ad.dispatch_async_delegation( goal="g", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=runner, max_async_children=3, ) elapsed = time.monotonic() - t0 assert res["status"] == "dispatched" assert res["delegation_id"].startswith("deleg_") # Non-blocking invariant: dispatch returned while the runner is still # gated (active), so it cannot have waited on the gate. The active_count # check is the environment-independent proof; the generous wall-clock # bound is a loose sanity backstop, not the primary assertion (a loaded # CI runner can be slow but never anywhere near the runner's 5s gate). assert ad.active_count() == 1 assert elapsed < 4.0, f"dispatch blocked {elapsed:.2f}s (gate is 5s)" gate.set() def test_async_executor_workers_are_daemon_threads(): gate = threading.Event() def runner(): gate.wait(timeout=60) return {"status": "completed", "summary": "done"} res = ad.dispatch_async_delegation( goal="daemon check", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=runner, max_async_children=1, ) assert res["status"] == "dispatched" deadline = time.monotonic() + 2 worker = None while time.monotonic() < deadline: worker = next( (t for t in threading.enumerate() if t.name.startswith("async-delegate")), None, ) if worker is not None: break time.sleep(0.02) assert worker is not None assert worker.daemon is True gate.set() assert _drain_one() is not None def test_completion_event_lands_on_shared_queue_with_session_key(): def runner(): return {"status": "completed", "summary": "the result", "api_calls": 3, "duration_seconds": 2.0, "model": "test-model"} res = ad.dispatch_async_delegation( goal="compute X", context="some context", toolsets=["web", "file"], role="leaf", model="test-model", session_key="agent:main:cli:dm:local", parent_session_id="20260703_parent_sid", runner=runner, max_async_children=3, ) assert res["status"] == "dispatched" evt = _drain_one() assert evt is not None assert evt["type"] == "async_delegation" assert evt["summary"] == "the result" assert evt["session_key"] == "agent:main:cli:dm:local" assert evt["parent_session_id"] == "20260703_parent_sid" assert evt["delegation_id"] == res["delegation_id"] def test_rich_reinjection_block_is_self_contained(): def runner(): return {"status": "completed", "summary": "The answer is 42.", "api_calls": 7, "duration_seconds": 3.5, "model": "test-model"} ad.dispatch_async_delegation( goal="Compute the meaning of life", context="User is a philosopher. Respond tersely.", toolsets=["web"], role="leaf", model="test-model", session_key="", runner=runner, max_async_children=3, ) evt = _drain_one() assert evt is not None text = format_process_notification(evt) assert text is not None for needle in [ "ASYNC DELEGATION COMPLETE", "Compute the meaning of life", "User is a philosopher", "Toolsets: web", "The answer is 42.", "Status: completed", "API calls: 7", ]: assert needle in text, f"missing {needle!r}" def test_dispatch_rejected_at_capacity(): ev = threading.Event() def blocker(): ev.wait(timeout=60) return {"status": "completed", "summary": "x"} for i in range(2): r = ad.dispatch_async_delegation( goal=f"task{i}", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=blocker, max_async_children=2, ) assert r["status"] == "dispatched" r3 = ad.dispatch_async_delegation( goal="task3", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=blocker, max_async_children=2, ) assert r3["status"] == "rejected" assert "capacity reached" in r3["error"] ev.set() def test_interrupt_all_signals_running_children(): ev = threading.Event() interrupted = {"count": 0} # No short internal timeout: the blocker holds until interrupt_fn fires. # The old ev.wait(timeout=5) made this test a change-detector for CI # worker load — on a CPU-starved runner the 5s expired before # interrupt_all() ran, the record finalized, and interrupt_all() found # nothing running (n == 0). The pytest-level timeout is the real # runaway guard. def blocker(): ev.wait(timeout=60) return {"status": "interrupted", "summary": None, "error": "cancelled"} def interrupt_fn(): interrupted["count"] += 1 ev.set() r = ad.dispatch_async_delegation( goal="long task", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=blocker, interrupt_fn=interrupt_fn, max_async_children=3, ) n = ad.interrupt_all(reason="test") assert n == 1 assert interrupted["count"] == 1 # child still emits a completion event after interrupt. Match on THIS # delegation's id — straggler 'completed' events from a previous test's # workers can finalize after that test's teardown drain and leak into # this queue (observed on loaded CI workers). evt = _drain_for(r["delegation_id"]) assert evt is not None assert evt["status"] == "interrupted" def _fast_stale_monitor(monkeypatch, *, idle=0.15, in_tool=0.3, grace=0.15): """Shrink the stale-monitor cadence so tests run in milliseconds.""" monkeypatch.setattr(ad, "_STALE_CHECK_INTERVAL", 0.03) monkeypatch.setattr(ad, "_STALE_IDLE_SECONDS", idle) monkeypatch.setattr(ad, "_STALE_IN_TOOL_SECONDS", in_tool) monkeypatch.setattr(ad, "_STALL_GRACE_SECONDS", grace) def test_stalled_runner_is_interrupted_then_finalized(monkeypatch): _fast_stale_monitor(monkeypatch) gate = threading.Event() interrupted = {"count": 0} def stuck_runner(): gate.wait(timeout=10) return {"status": "completed", "summary": "too late"} def interrupt_fn(): interrupted["count"] += 1 res = ad.dispatch_async_delegation( goal="stuck child", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=stuck_runner, interrupt_fn=interrupt_fn, max_async_children=1, # Frozen progress token: the child never advances an API call. progress_fn=lambda: ((0, None), False), ) assert res["status"] == "dispatched" evt = _drain_for(res["delegation_id"], timeout=5.0) try: assert evt is not None assert evt["type"] == "async_delegation" assert evt["status"] == "stalled" assert evt["delegation_id"] == res["delegation_id"] assert evt["api_calls"] == 0 assert "stalled" in evt["error"] # Interrupt was requested BEFORE force-finalization (grace window). assert interrupted["count"] >= 1 assert ad.active_count() == 0 finally: gate.set() # If the ignored runner eventually returns, it must not enqueue a second # completion for a delegation the monitor already finalized. assert _drain_one(timeout=0.5) is None def test_progressing_runner_is_never_stalled(monkeypatch): """A child that keeps advancing is left alone no matter how long it runs.""" _fast_stale_monitor(monkeypatch) gate = threading.Event() ticks = {"n": 0} def slow_but_alive_runner(): gate.wait(timeout=10) return {"status": "completed", "summary": "done", "api_calls": 7} def progress_fn(): # Token advances on every sample — simulates a child making steady # API-call progress. ticks["n"] += 1 return (ticks["n"], None), False res = ad.dispatch_async_delegation( goal="slow child", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=slow_but_alive_runner, max_async_children=1, progress_fn=progress_fn, ) assert res["status"] == "dispatched" # Run well past the (shrunk) idle threshold — several monitor sweeps. time.sleep(0.6) assert ad.active_count() == 1 assert process_registry.completion_queue.empty() gate.set() evt = _drain_for(res["delegation_id"], timeout=5.0) assert evt is not None assert evt["status"] == "completed" assert evt["summary"] == "done" def test_stalling_runner_that_honors_interrupt_keeps_its_result(monkeypatch): """Interrupt-responsive children finalize through the NORMAL path. The monitor's interrupt gives a wedged-looking child a grace window; if the runner returns during it, the real result (partial work, api_calls) is delivered instead of a synthetic stalled event. """ _fast_stale_monitor(monkeypatch, grace=5.0) interrupted = threading.Event() def runner(): # "Wedged" until interrupted, then unwinds and reports partial work. interrupted.wait(timeout=10) return { "status": "interrupted", "summary": "partial work saved", "api_calls": 3, } res = ad.dispatch_async_delegation( goal="responsive child", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=runner, interrupt_fn=interrupted.set, max_async_children=1, progress_fn=lambda: ((3, None), False), ) assert res["status"] == "dispatched" evt = _drain_for(res["delegation_id"], timeout=5.0) assert evt is not None assert evt["status"] == "interrupted" assert evt["summary"] == "partial work saved" assert evt["api_calls"] == 3 assert ad.active_count() == 0 def test_streaming_child_counts_as_alive(monkeypatch): """A child mid-stream (api_call_count frozen, last_activity_ts ticking) must never be stalled — streamed chunks tick _touch_activity, and the progress token includes that timestamp (same liveness signal as the compaction inactivity budget, PR #71508).""" _fast_stale_monitor(monkeypatch) gate = threading.Event() now = {"ts": 1000.0} def progress_fn(): # api_call_count and current_tool frozen (long streaming response in # flight), but the activity timestamp advances with every chunk. now["ts"] += 1.0 return ((1, None, now["ts"]),), False res = ad.dispatch_async_delegation( goal="streaming child", context=None, toolsets=None, role="leaf", model="m", session_key="", max_async_children=1, runner=lambda: (gate.wait(timeout=10), {"status": "completed", "summary": "streamed"})[1], progress_fn=progress_fn, ) assert res["status"] == "dispatched" time.sleep(0.6) # several sweeps past the shrunk idle threshold assert ad.active_count() == 1 assert process_registry.completion_queue.empty() gate.set() evt = _drain_for(res["delegation_id"], timeout=5.0) assert evt is not None assert evt["status"] == "completed" def test_stalled_event_carries_structured_stall_metadata(monkeypatch): """The terminal stalled event must expose machine-readable stall context (#51690) — quiet duration, tripped threshold, phase, grace — mirroring the sync path's timeout_seconds/timed_out_after_seconds/timeout_phase.""" _fast_stale_monitor(monkeypatch) gate = threading.Event() res = ad.dispatch_async_delegation( goal="stall metadata", context=None, toolsets=None, role="leaf", model="m", session_key="", max_async_children=1, runner=lambda: {} if gate.wait(timeout=10) else {}, progress_fn=lambda: ((0, "terminal"), True), ) assert res["status"] == "dispatched" evt = _drain_for(res["delegation_id"], timeout=5.0) try: assert evt is not None assert evt["status"] == "stalled" assert evt["stalled_after_quiet_seconds"] >= 0.3 # in-tool threshold assert evt["stall_threshold_seconds"] == ad._STALE_IN_TOOL_SECONDS assert evt["stall_phase"] == "in_tool" assert evt["stall_grace_seconds"] == ad._STALL_GRACE_SECONDS finally: gate.set() def test_list_async_delegations_exposes_live_activity(monkeypatch): """list_async_delegations must expose per-child live activity sampled from progress_fn plus seconds_since_progress, for /agents UIs (#51690).""" monkeypatch.setattr(ad, "_STALE_CHECK_INTERVAL", 0.03) gate = threading.Event() base_ts = time.time() - 12.0 res = ad.dispatch_async_delegation( goal="live listing", context=None, toolsets=None, role="leaf", model="m", session_key="", max_async_children=1, runner=lambda: {} if gate.wait(timeout=10) else {}, progress_fn=lambda: (((3, "web_search", base_ts),), True), ) try: time.sleep(0.1) # let the monitor stamp _progress_ts at least once item = next( d for d in ad.list_async_delegations() if d["delegation_id"] == res["delegation_id"] ) assert item["status"] == "running" assert item["in_tool"] is True assert "seconds_since_progress" in item (child,) = item["children_activity"] assert child["api_calls"] == 3 assert child["current_tool"] == "web_search" assert 10.0 <= child["seconds_since_activity"] <= 20.0 # Callables and private bookkeeping must never leak. assert "progress_fn" not in item assert "interrupt_fn" not in item assert not any(k.startswith("_") for k in item) finally: gate.set() def test_stalled_batch_is_interrupted_then_finalized(monkeypatch): _fast_stale_monitor(monkeypatch) gate = threading.Event() interrupted = {"count": 0} def stuck_batch(): gate.wait(timeout=10) return {"results": [{"status": "completed", "summary": "too late"}]} def interrupt_fn(): interrupted["count"] += 1 res = ad.dispatch_async_delegation_batch( goals=["a", "b"], context="ctx", toolsets=None, role="leaf", model="m", session_key="", runner=stuck_batch, interrupt_fn=interrupt_fn, max_async_children=1, progress_fn=lambda: (((0, None), (0, None)), False), ) assert res["status"] == "dispatched" evt = _drain_for(res["delegation_id"], timeout=5.0) try: assert evt is not None assert evt["type"] == "async_delegation" assert evt["status"] == "stalled" assert evt["is_batch"] is True assert evt["goals"] == ["a", "b"] assert evt["results"] == [] assert "stalled" in evt["error"] assert interrupted["count"] >= 1 assert ad.active_count() == 0 finally: gate.set() assert _drain_one(timeout=0.5) is None def test_in_tool_stall_uses_higher_threshold(monkeypatch): """A frozen child inside a tool gets the in-tool ceiling, not the idle one.""" _fast_stale_monitor(monkeypatch, idle=0.1, in_tool=10.0, grace=0.1) gate = threading.Event() def runner(): gate.wait(timeout=10) return {"status": "completed", "summary": "long tool finished"} res = ad.dispatch_async_delegation( goal="long tool child", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=runner, max_async_children=1, # Frozen token but in_tool=True — a legitimately slow terminal # command / web fetch. Must NOT be stalled at the idle threshold. progress_fn=lambda: ((1, "terminal"), True), ) assert res["status"] == "dispatched" time.sleep(0.5) # far past idle threshold, well under in-tool threshold assert ad.active_count() == 1 assert process_registry.completion_queue.empty() gate.set() evt = _drain_for(res["delegation_id"], timeout=5.0) assert evt is not None assert evt["status"] == "completed" def test_stall_stays_finalizing_until_durable_persistence(tmp_path, monkeypatch): _fast_stale_monitor(monkeypatch) gate = threading.Event() persist_entered = threading.Event() allow_persist = threading.Event() real_persist = ad._persist_completion def blocking_persist(event, result): persist_entered.set() allow_persist.wait(timeout=5) real_persist(event, result) def stuck_runner(): gate.wait(timeout=10) return {"status": "completed", "summary": "too late"} monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setattr(ad, "_persist_completion", blocking_persist) dispatched = ad.dispatch_async_delegation( goal="durable stall", context=None, toolsets=None, role="leaf", model="m", session_key="owner", runner=stuck_runner, max_async_children=1, progress_fn=lambda: ((0, None), False), ) try: assert persist_entered.wait(timeout=5) assert ad.active_count() == 1 record = next( item for item in ad.list_async_delegations() if item["delegation_id"] == dispatched["delegation_id"] ) assert record["status"] == "finalizing" assert process_registry.completion_queue.empty() allow_persist.set() evt = _drain_for(dispatched["delegation_id"]) assert evt is not None assert evt["status"] == "stalled" assert ad.active_count() == 0 durable = ad.get_durable_delegation(dispatched["delegation_id"]) assert durable["state"] == "stalled" assert durable["delivery_state"] == "pending" finally: allow_persist.set() gate.set() def test_stalled_completion_restores_once_after_process_restart(tmp_path): repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) env = {**os.environ, "HERMES_HOME": str(tmp_path), "PYTHONPATH": repo} producer = r''' import json import threading import time from tools import async_delegation as ad ad._STALE_CHECK_INTERVAL = 0.03 ad._STALE_IDLE_SECONDS = 0.1 ad._STALL_GRACE_SECONDS = 0.1 gate = threading.Event() r = ad.dispatch_async_delegation( goal="restart stall", context=None, toolsets=None, role="leaf", model="m", session_key="owner-session", parent_session_id="durable-parent", runner=lambda: gate.wait(timeout=60), progress_fn=lambda: ((0, None), False), ) deadline = time.time() + 10 while ad.active_count() and time.time() < deadline: time.sleep(.01) row = ad.get_durable_delegation(r["delegation_id"]) print(json.dumps({"delegation_id": r["delegation_id"], "row": row}, sort_keys=True)) ''' first = subprocess.run( [sys.executable, "-c", producer], cwd=repo, env=env, text=True, capture_output=True, timeout=30, check=True, ) produced = json.loads(first.stdout.strip().splitlines()[-1]) delegation_id = produced["delegation_id"] assert produced["row"]["state"] == "stalled" assert produced["row"]["delivery_state"] == "pending" consumer = r''' import json from tools.process_registry import process_registry evt = process_registry.completion_queue.get_nowait() print(json.dumps({"event": evt, "remaining": process_registry.completion_queue.qsize()}, sort_keys=True)) ''' second = subprocess.run( [sys.executable, "-c", consumer], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) restored = json.loads(second.stdout.strip().splitlines()[-1]) assert restored["remaining"] == 0 assert restored["event"]["delegation_id"] == delegation_id assert restored["event"]["status"] == "stalled" assert restored["event"]["restored"] is True acker = f''' from tools import async_delegation as ad assert ad.mark_completion_delivered({delegation_id!r}) ''' subprocess.run( [sys.executable, "-c", acker], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) probe = subprocess.run( [sys.executable, "-c", "from tools.process_registry import process_registry; print(process_registry.completion_queue.qsize())"], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) assert probe.stdout.strip().splitlines()[-1] == "0" def test_completed_records_pruned_to_cap(): # Run more than the retention cap quickly; ensure list doesn't grow forever. for i in range(ad._MAX_RETAINED_COMPLETED + 10): ad.dispatch_async_delegation( goal=f"t{i}", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=lambda: {"status": "completed", "summary": "ok"}, max_async_children=ad._MAX_RETAINED_COMPLETED + 20, ) # let workers finish deadline = time.monotonic() + 10 while time.monotonic() < deadline and ad.active_count() > 0: time.sleep(0.05) assert len(ad.list_async_delegations()) <= ad._MAX_RETAINED_COMPLETED def test_completion_is_persisted_and_delivery_can_be_acknowledged(tmp_path, monkeypatch): """A finished child remains pending on disk until its queue consumer acks it.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) dispatched = ad.dispatch_async_delegation( goal="durable", context="ctx", toolsets=["terminal"], role="leaf", model="m", session_key="owner", parent_session_id="parent", runner=lambda: {"status": "completed", "summary": "survived"}, ) assert _drain_one() is not None restored = queue.Queue() assert ad.restore_undelivered_completions(restored) == 1 row = ad.get_durable_delegation(dispatched["delegation_id"]) assert row["origin_session"] == "owner" assert row["state"] == "completed" assert row["result"]["summary"] == "survived" assert row["delivery_state"] == "pending" # Queue publication/restoration is not a destination delivery attempt. assert row["delivery_attempts"] == 0 assert ad.mark_completion_delivered(dispatched["delegation_id"]) assert ad.restore_undelivered_completions(queue.Queue()) == 0 assert ad.get_durable_delegation(dispatched["delegation_id"])["delivery_state"] == "delivered" def test_real_process_restart_restores_owned_completion_once(tmp_path): """Real-import E2E: a fresh interpreter restores a prior process's result.""" repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) env = {**os.environ, "HERMES_HOME": str(tmp_path), "PYTHONPATH": repo} producer = r''' import time from tools import async_delegation as ad r = ad.dispatch_async_delegation( goal="restart", context=None, toolsets=None, role="leaf", model="m", session_key="owner-session", parent_session_id="durable-parent", runner=lambda: {"status": "completed", "summary": "after restart"}, ) deadline = time.time() + 5 while ad.active_count() and time.time() < deadline: time.sleep(.01) print(r["delegation_id"]) ''' first = subprocess.run( [sys.executable, "-c", producer], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) delegation_id = first.stdout.strip().splitlines()[-1] consumer = r''' import json from tools.process_registry import process_registry evt = process_registry.completion_queue.get_nowait() print(json.dumps(evt, sort_keys=True)) ''' second = subprocess.run( [sys.executable, "-c", consumer], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) evt = json.loads(second.stdout.strip().splitlines()[-1]) assert evt["delegation_id"] == delegation_id assert evt["session_key"] == "owner-session" assert evt["parent_session_id"] == "durable-parent" assert evt["summary"] == "after restart" acker = f''' from tools import async_delegation as ad assert ad.mark_completion_delivered({delegation_id!r}) ''' subprocess.run( [sys.executable, "-c", acker], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) probe = subprocess.run( [sys.executable, "-c", "from tools.process_registry import process_registry; print(process_registry.completion_queue.qsize())"], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) assert probe.stdout.strip().splitlines()[-1] == "0" def test_submit_failure_removes_durable_running_record(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) class _BrokenExecutor: def submit(self, *_args, **_kwargs): raise RuntimeError("submit failed") monkeypatch.setattr(ad, "_get_executor", lambda _max_workers: _BrokenExecutor()) result = ad.dispatch_async_delegation( goal="never ran", context=None, toolsets=None, role="leaf", model="m", session_key="owner", runner=lambda: {}, ) assert result["status"] == "rejected" with ad._DB_LOCK, ad._connect() as conn: assert conn.execute("SELECT COUNT(*) FROM async_delegations").fetchone()[0] == 0 def test_pending_retention_prunes_delivered_before_undelivered(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setattr(ad, "_MAX_RETAINED_COMPLETED", 2) for index, delivery_state in enumerate(("pending", "delivered", "pending")): delegation_id = f"deleg_{index}" record = { "delegation_id": delegation_id, "session_key": "owner", "origin_ui_session_id": "", "parent_session_id": None, "dispatched_at": float(index + 1), } ad._persist_dispatch(record) ad._persist_completion( { "delegation_id": delegation_id, "status": "completed", "completed_at": float(index + 1), }, {"status": "completed", "summary": delegation_id}, ) if delivery_state == "delivered": ad.mark_completion_delivered(delegation_id) ad._prune_durable_records() assert ad.get_durable_delegation("deleg_0") is not None assert ad.get_durable_delegation("deleg_1") is None assert ad.get_durable_delegation("deleg_2") is not None def test_recover_marks_abandoned_running_record_unknown(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) record = { "delegation_id": "deleg_abandoned", "session_key": "owner", "origin_ui_session_id": "", "parent_session_id": None, "dispatched_at": 1.0, } ad._persist_dispatch(record) with ad._DB_LOCK, ad._connect() as conn: conn.execute( "UPDATE async_delegations SET owner_pid=?, owner_started_at=NULL WHERE delegation_id=?", (99999999, "deleg_abandoned"), ) assert ad.recover_abandoned_delegations() == 1 durable = ad.get_durable_delegation("deleg_abandoned") assert durable["state"] == "unknown" assert durable["delivery_state"] == "pending" restored = queue.Queue() assert ad.restore_undelivered_completions(restored) == 1 assert restored.get_nowait()["status"] == "unknown" def test_origin_session_id_survives_persistence_round_trip(tmp_path, monkeypatch): """origin_session_id (the api_server wake self-post target) must be persisted with the durable dispatch record and restored on recovery — otherwise completions recovered after a process restart are unroutable to api_server sessions (in-memory record is gone).""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) record = { "delegation_id": "deleg_wake_target", "session_key": "owner", "origin_ui_session_id": "", "origin_session_id": "raw-api-sid-42", "parent_session_id": None, "dispatched_at": 1.0, } ad._persist_dispatch(record) # Durable record carries the wake target. durable = ad.get_durable_delegation("deleg_wake_target") assert durable["origin_session_id"] == "raw-api-sid-42" # Simulate the owning process dying, then recovery after restart: the # regenerated completion event must still carry the wake target. with ad._DB_LOCK, ad._connect() as conn: conn.execute( "UPDATE async_delegations SET owner_pid=?, owner_started_at=NULL WHERE delegation_id=?", (99999999, "deleg_wake_target"), ) restored = queue.Queue() assert ad.restore_undelivered_completions(restored) == 1 evt = restored.get_nowait() assert evt["delegation_id"] == "deleg_wake_target" assert evt["origin_session_id"] == "raw-api-sid-42" assert evt["restored"] is True def test_origin_session_id_migration_backfills_legacy_rows(tmp_path, monkeypatch): """Rows written by a pre-origin_session_id build must survive the ALTER TABLE migration and read back as an empty wake target.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) # Create a legacy-schema DB (no origin_session_id column). import sqlite3 db_path = ad._db_path() db_path.parent.mkdir(parents=True, exist_ok=True) legacy = sqlite3.connect(str(db_path)) legacy.execute( """CREATE TABLE async_delegations ( delegation_id TEXT PRIMARY KEY, origin_session TEXT NOT NULL, origin_ui_session_id TEXT NOT NULL DEFAULT '', parent_session_id TEXT, state TEXT NOT NULL, dispatched_at REAL NOT NULL, completed_at REAL, updated_at REAL NOT NULL, event_json TEXT, result_json TEXT, delivery_state TEXT NOT NULL DEFAULT 'pending', delivery_attempts INTEGER NOT NULL DEFAULT 0, delivered_at REAL )""" ) legacy.execute( """INSERT INTO async_delegations (delegation_id, origin_session, state, dispatched_at, updated_at) VALUES ('deleg_legacy', 'owner', 'running', 1.0, 1.0)""" ) legacy.commit() legacy.close() durable = ad.get_durable_delegation("deleg_legacy") assert durable is not None assert durable["origin_session_id"] == "" def test_durable_delivery_claim_is_exclusive_and_retryable(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) record = { "delegation_id": "deleg_claim", "session_key": "owner", "origin_ui_session_id": "", "parent_session_id": None, "dispatched_at": 1.0, } ad._persist_dispatch(record) ad._persist_completion( {"delegation_id": "deleg_claim", "status": "completed", "completed_at": 2.0}, {"status": "completed", "summary": "done"}, ) assert ad.claim_completion_delivery("deleg_claim", "consumer-a") assert not ad.claim_completion_delivery("deleg_claim", "consumer-b") assert ad.release_completion_delivery("deleg_claim", "consumer-a") assert ad.claim_completion_delivery("deleg_claim", "consumer-b") assert ad.complete_completion_delivery("deleg_claim", "consumer-b") assert not ad.claim_completion_delivery("deleg_claim", "consumer-c") assert ad.get_durable_delegation("deleg_claim")["delivery_state"] == "delivered" # --------------------------------------------------------------------------- # Integration: delegate_task(background=True) routing # --------------------------------------------------------------------------- def test_delegate_task_background_routes_async_and_does_not_block(monkeypatch): """delegate_task(background=True) returns a handle without running the child synchronously, and the child completes on the background thread. A single task is dispatched as a one-item background batch unit.""" from unittest.mock import MagicMock, patch import tools.delegate_tool as dt parent = MagicMock() parent._delegate_depth = 0 parent.session_id = "sess" parent._interrupt_requested = False parent._active_children = [] parent._active_children_lock = None fake_child = MagicMock() fake_child._delegate_role = "leaf" fake_child._subagent_id = "s1" gate = threading.Event() def slow_child(task_index, goal, child=None, parent_agent=None, **kw): gate.wait(timeout=60) # a sync impl would hang delegate_task here return { "task_index": 0, "status": "completed", "summary": f"done: {goal}", "api_calls": 1, "duration_seconds": 0.1, "model": "m", "exit_reason": "completed", } creds = { "model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None, } # monkeypatch (not `with`) so patches outlive delegate_task's return and # remain active while the background worker runs. monkeypatch.setattr(dt, "_build_child_agent", lambda **kw: fake_child) monkeypatch.setattr(dt, "_run_single_child", slow_child) monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: creds) out = dt.delegate_task( goal="the real task", context="ctx", background=True, parent_agent=parent, ) import json parsed = json.loads(out) assert parsed["status"] == "dispatched" assert parsed["mode"] == "background" assert parsed["delegation_id"].startswith("deleg_") # Non-blocking invariant: delegate_task returned while the child is STILL # blocked on the closed gate, so no completion event exists yet. assert process_registry.completion_queue.empty() assert ad.active_count() == 1 # one background batch unit, not finished gate.set() evt = _drain_one() assert evt is not None assert evt["type"] == "async_delegation" # Single task rides the batch path → carries a 1-item results list. assert evt.get("is_batch") is True assert len(evt["results"]) == 1 assert evt["results"][0]["summary"] == "done: the real task" text = format_process_notification(evt) assert text is not None assert "the real task" in text def test_delegate_task_background_waits_inside_kanban_worker(monkeypatch): """A dispatcher-spawned Kanban worker is a finite process, so a required delegated result must return in-turn instead of becoming an orphaned background completion after the parent exits.""" import json from unittest.mock import MagicMock import tools.delegate_tool as dt monkeypatch.setenv("HERMES_KANBAN_TASK", "t_review") parent = MagicMock() parent._delegate_depth = 0 parent.session_id = "kanban-worker-session" parent._interrupt_requested = False parent._active_children = [] parent._active_children_lock = None fake_child = MagicMock() fake_child._delegate_role = "leaf" started = threading.Event() release = threading.Event() def delayed_child(task_index, goal, child=None, parent_agent=None, **kw): started.set() release.wait(timeout=5) return { "task_index": task_index, "status": "completed", "summary": "review approved", "api_calls": 1, "duration_seconds": 0.1, "model": "m", "exit_reason": "completed", } creds = { "model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None, } monkeypatch.setattr(dt, "_build_child_agent", lambda **kw: fake_child) monkeypatch.setattr(dt, "_run_single_child", delayed_child) monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: creds) captured = {} def call_delegate(): captured["output"] = dt.delegate_task( goal="independent review", background=True, parent_agent=parent, ) caller = threading.Thread(target=call_delegate) caller.start() assert started.wait(timeout=2) assert caller.is_alive(), "Kanban delegate_task returned before its child finished" assert ad.active_count() == 0 release.set() caller.join(timeout=5) assert not caller.is_alive() parsed = json.loads(captured["output"]) assert parsed["results"][0]["summary"] == "review approved" assert "SYNCHRONOUSLY" in parsed["note"] assert process_registry.completion_queue.empty() def test_delegate_task_background_uses_live_tui_agent_session_id(monkeypatch): """TUI async delegation must route to the live/compressed agent id. Regression: delegate_task captured the stale approval/session context key after compression rotated parent_agent.session_id. The resulting completion was orphaned and could be consumed by an unrelated desktop session poller. """ import json from unittest.mock import MagicMock import tools.delegate_tool as dt from gateway.session_context import clear_session_vars, set_session_vars from tools.approval import reset_current_session_key, set_current_session_key parent = MagicMock() parent._delegate_depth = 0 parent.session_id = "post-compress-tip" parent._interrupt_requested = False parent._active_children = [] parent._active_children_lock = None fake_child = MagicMock() fake_child._delegate_role = "leaf" creds = { "model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None, } monkeypatch.setattr(dt, "_build_child_agent", lambda **kw: fake_child) monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: creds) monkeypatch.setattr( dt, "_run_single_child", lambda *a, **k: { "task_index": 0, "status": "completed", "summary": "done", "api_calls": 1, "duration_seconds": 0.1, "model": "m", "exit_reason": "completed", }, ) approval_token = set_current_session_key("pre-compress-parent") session_tokens = set_session_vars( source="tui", session_key="pre-compress-parent", ui_session_id="origin-tab", ) try: out = dt.delegate_task(goal="bg task", background=True, parent_agent=parent) assert json.loads(out)["status"] == "dispatched" evt = _drain_one() finally: reset_current_session_key(approval_token) clear_session_vars(session_tokens) assert evt is not None assert evt["type"] == "async_delegation" assert evt["session_key"] == "post-compress-tip" assert evt["origin_ui_session_id"] == "origin-tab" def test_delegate_task_background_batch_runs_as_one_unit(monkeypatch): """A multi-item batch with background=True dispatches the WHOLE fan-out as ONE background unit (one handle, one async slot). The children run in parallel and join; the consolidated results come back as a single completion event when ALL of them finish.""" import json from unittest.mock import MagicMock, patch import tools.delegate_tool as dt parent = MagicMock() parent._delegate_depth = 0 parent.session_id = "sess" parent._interrupt_requested = False parent._active_children = [] parent._active_children_lock = None fake_child = MagicMock() fake_child._delegate_role = "leaf" gate = threading.Event() def _blocking_child(task_index, goal, child=None, parent_agent=None, **kw): gate.wait(timeout=60) return { "task_index": task_index, "status": "completed", "summary": f"done: {goal}", "api_calls": 1, "duration_seconds": 0.1, "model": "m", "exit_reason": "completed", } creds = { "model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None, } # Use monkeypatch (not a `with` block) so the patches stay active while the # background worker thread runs _execute_and_aggregate AFTER delegate_task # has already returned. monkeypatch.setattr(dt, "_build_child_agent", lambda **kw: fake_child) monkeypatch.setattr(dt, "_run_single_child", _blocking_child) monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: creds) out = dt.delegate_task( tasks=[{"goal": "a"}, {"goal": "b"}, {"goal": "c"}], background=True, parent_agent=parent, ) parsed = json.loads(out) assert parsed["status"] == "dispatched" assert parsed["mode"] == "background" assert parsed["count"] == 3 assert parsed["delegation_id"].startswith("deleg_") assert parsed["goals"] == ["a", "b", "c"] # ONE background unit for the whole fan-out (not three), and the call # returned while all children are still blocked → chat not blocked. assert process_registry.completion_queue.empty() assert ad.active_count() == 1 # Release the children; the whole batch joins and emits ONE event. gate.set() evt = _drain_one() assert evt is not None assert evt["type"] == "async_delegation" assert evt.get("is_batch") is True assert len(evt["results"]) == 3 summaries = sorted(r["summary"] for r in evt["results"]) assert summaries == ["done: a", "done: b", "done: c"] # The consolidated notification names all three tasks in one block. text = format_process_notification(evt) assert text is not None assert "TASK 1/3" in text and "TASK 2/3" in text and "TASK 3/3" in text assert "done: a" in text and "done: b" in text and "done: c" in text # No more events — it's a single combined completion, not N of them. assert _drain_one() is None def test_delegate_task_background_passes_progress_fn_to_async_registry(monkeypatch): import json from unittest.mock import MagicMock import tools.delegate_tool as dt parent = MagicMock() parent._delegate_depth = 0 parent.session_id = "sess" parent._interrupt_requested = False parent._active_children = [] parent._active_children_lock = None fake_child = MagicMock() fake_child._delegate_role = "leaf" fake_child._subagent_id = "s1" fake_child.get_activity_summary.return_value = { "api_call_count": 4, "current_tool": "terminal", "last_activity_ts": 1234.5, } creds = { "model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None, } captured = {} def fake_dispatch(**kwargs): captured.update(kwargs) return {"status": "dispatched", "delegation_id": "deleg_progress"} monkeypatch.setattr(dt, "_build_child_agent", lambda **kw: fake_child) monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: creds) monkeypatch.setattr(ad, "dispatch_async_delegation_batch", fake_dispatch) out = dt.delegate_task(goal="background stall guard", background=True, parent_agent=parent) parsed = json.loads(out) assert parsed["status"] == "dispatched" assert parsed["delegation_id"] == "deleg_progress" # The dispatch wires a live progress sampler over the child agents so the # async registry's stale monitor can watch the detached batch. The token # includes last_activity_ts so streamed chunks count as liveness (each # chunk ticks _touch_activity), not just completed API calls. progress_fn = captured["progress_fn"] assert callable(progress_fn) token, in_tool = progress_fn() assert token == ((4, "terminal", 1234.5),) assert in_tool is True def test_model_dispatch_forces_background(): """The MODEL-facing dispatch path forces background=True for any top-level delegation (single task OR batch), and keeps it off for an orchestrator subagent (depth > 0). Direct delegate_task() callers are unaffected (they keep the synchronous default).""" import tools.delegate_tool as dt from unittest.mock import MagicMock top = MagicMock() top._delegate_depth = 0 sub = MagicMock() sub._delegate_depth = 1 # Registry-fallback helper: top-level always background, regardless of # single vs batch; subagent never. assert dt._model_background_value({"goal": "x"}, top) is True assert dt._model_background_value( {"tasks": [{"goal": "a"}, {"goal": "b"}]}, top ) is True assert dt._model_background_value({"tasks": [{"goal": "a"}]}, top) is True assert dt._model_background_value({"goal": "x"}, sub) is False assert dt._model_background_value( {"tasks": [{"goal": "a"}, {"goal": "b"}]}, sub ) is False def test_run_agent_dispatch_forces_background(): """run_agent._dispatch_delegate_task — the live model path — forces background on for any top-level delegation (single OR batch) and off for a subagent.""" from unittest.mock import patch import run_agent class _FakeAgent: _delegate_depth = 0 captured = {} def _fake_delegate(**kwargs): captured.update(kwargs) return "{}" with patch("tools.delegate_tool.delegate_task", _fake_delegate): agent = _FakeAgent() run_agent.AIAgent._dispatch_delegate_task(agent, {"goal": "x"}) assert captured["background"] is True run_agent.AIAgent._dispatch_delegate_task( agent, {"tasks": [{"goal": "a"}, {"goal": "b"}]} ) assert captured["background"] is True sub = _FakeAgent() sub._delegate_depth = 1 run_agent.AIAgent._dispatch_delegate_task(sub, {"goal": "x"}) assert captured["background"] is False def test_dispatch_never_forwards_model_toolsets(): """The model has no toolsets argument — subagents always inherit the parent's toolsets. Even if a model smuggles a `toolsets` key into the tool-call args, the live dispatch path must NOT forward it to delegate_task (which no longer accepts it) and must not crash.""" from unittest.mock import patch import run_agent class _FakeAgent: _delegate_depth = 0 captured = {} def _fake_delegate(**kwargs): captured.update(kwargs) return "{}" with patch("tools.delegate_tool.delegate_task", _fake_delegate): run_agent.AIAgent._dispatch_delegate_task( _FakeAgent(), {"goal": "x", "toolsets": ["web", "terminal"]} ) assert "toolsets" not in captured def test_delegate_task_background_detaches_child_from_parent(monkeypatch): """A background child must NOT remain in parent._active_children — otherwise parent-turn interrupts / cache evicts / session close would kill the detached subagent mid-run.""" from unittest.mock import MagicMock, patch import tools.delegate_tool as dt parent = MagicMock() parent._delegate_depth = 0 parent.session_id = "sess" parent._active_children = [] parent._active_children_lock = threading.Lock() fake_child = MagicMock() fake_child._delegate_role = "leaf" fake_child._subagent_id = "s1" gate = threading.Event() def slow_child(task_index, goal, child=None, parent_agent=None, **kw): gate.wait(timeout=60) return {"task_index": 0, "status": "completed", "summary": "ok"} def build_and_register(**kw): # Mirror what the real _build_child_agent does: register the child # for interrupt propagation. parent._active_children.append(fake_child) return fake_child creds = { "model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None, } with patch.object(dt, "_build_child_agent", side_effect=build_and_register), \ patch.object(dt, "_run_single_child", side_effect=slow_child), \ patch.object(dt, "_resolve_delegation_credentials", return_value=creds): out = dt.delegate_task(goal="bg task", background=True, parent_agent=parent) import json assert json.loads(out)["status"] == "dispatched" # Child detached immediately at dispatch, while it is still running. assert fake_child not in parent._active_children gate.set() assert _drain_one() is not None def test_concurrent_dispatch_respects_capacity(): """Two threads racing dispatch with cap=1 must yield exactly one accept (capacity check and record insert are atomic under the records lock).""" gate = threading.Event() def blocker(): gate.wait(timeout=60) return {"status": "completed", "summary": "x"} results = [] barrier = threading.Barrier(2) def racer(): barrier.wait(timeout=5) results.append( ad.dispatch_async_delegation( goal="race", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=blocker, max_async_children=1, ) ) threads = [threading.Thread(target=racer) for _ in range(2)] for t in threads: t.start() for t in threads: t.join(timeout=10) statuses = sorted(r["status"] for r in results) assert statuses == ["dispatched", "rejected"] gate.set() # --------------------------------------------------------------------------- # Gateway routing: session_key -> platform/chat_id, rich formatting, injection # --------------------------------------------------------------------------- def _make_async_evt(**over): evt = { "type": "async_delegation", "delegation_id": "deleg_x1", "session_key": "agent:main:telegram:dm:12345:678", "goal": "Investigate flaky test", "context": "repo /tmp/p", "toolsets": ["terminal"], "role": "leaf", "model": "m", "status": "completed", "summary": "Found the bug in test_foo", "api_calls": 4, "duration_seconds": 12.0, "dispatched_at": 1000.0, "completed_at": 1012.0, } evt.update(over) return evt def test_gateway_enriches_routing_from_session_key(): from gateway.run import GatewayRunner runner = object.__new__(GatewayRunner) evt = _make_async_evt() runner._enrich_async_delegation_routing(evt) assert evt["platform"] == "telegram" assert evt["chat_id"] == "12345" assert evt["thread_id"] == "678" def test_gateway_formatter_renders_async_block(): from gateway.run import _format_gateway_process_notification txt = _format_gateway_process_notification(_make_async_evt()) assert txt is not None assert "ASYNC DELEGATION COMPLETE" in txt assert "Found the bug in test_foo" in txt assert "Investigate flaky test" in txt def test_gateway_watch_drain_requeues_async_without_looping(): from gateway.run import _drain_gateway_watch_events q = queue.Queue() async_evt = _make_async_evt() watch_evt = { "type": "watch_match", "session_id": "proc_1", "command": "pytest", "pattern": "READY", "output": "READY", } q.put(async_evt) q.put(watch_evt) watch_events = _drain_gateway_watch_events(q) assert watch_events == [watch_evt] assert q.qsize() == 1 assert q.get_nowait() == async_evt def test_gateway_builds_routable_source_from_enriched_event(): from gateway.run import GatewayRunner runner = object.__new__(GatewayRunner) evt = _make_async_evt() runner._enrich_async_delegation_routing(evt) src = runner._build_process_event_source(evt) assert src is not None assert src.platform.value == "telegram" assert src.chat_id == "12345" def test_gateway_cli_origin_event_left_unrouted(): """An empty session_key (CLI origin) is left without routing fields.""" from gateway.run import GatewayRunner runner = object.__new__(GatewayRunner) evt = _make_async_evt(session_key="") runner._enrich_async_delegation_routing(evt) assert "platform" not in evt