mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-31 19:16:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

test: de-flake 30 timing-sensitive test files for loaded CI runners

Browse source 
Root-cause fixes from the flake audit (session-DB mining + repo sweep):

Event-based sync instead of sleep-sync:
- title_generator: mock sets threading.Event, wait(10) replaces
  sleep(0.3) hoping the daemon thread got scheduled
- docker zombie_reaping / profile_gateway: poll-for-state helpers
  replace fixed 1-3s sleeps (s6 transitions + SIGCHLD reaping are async)
- process_registry tree test: select()-bounded readline replaces an
  unbounded blocking read (parent wedge now fails THIS test with a clear
  message instead of an opaque rc=124 file kill); SIGTERM grace 1s->2s
  (the 1s partition window mid-interpreter-startup is how a child PID
  escaped the live-system guard in CI)

Timeout raises (loaded 8-way-sliced runners see ~5s scheduling floors;
all of these complete in ms-to-1s when healthy so the raises cost
nothing on green runs):
- subprocess/thread waits <= 2s raised to 10-15s across mcp_tool,
  mcp_circuit_breaker, mcp_reconnect_retry_reset, mcp_parked_self_probe,
  mcp_cancelled_error_propagation, registry, clarify_gateway, interrupt,
  voice_cli_integration, docker_environment, session_store_lock_io,
  planned_stop_watcher, cli_interrupt_subagent, thread_scoped_output
  (joins now also assert not is_alive() so stragglers fail loudly)
- wall-clock discrimination ceilings loosened where the guarded hang is
  10x larger: local_background_child_hang 4s->10s, interrupt_cleanup
  setup 5s->20s + pgid-exit 30s->60s, mcp_stability grandchild spinup
  5s->15s, protocol/gil-starvation fast-handler 0.5s->2s,
  iso_certify_seam 1.5s->5s, wait_for_mcp_discovery 0.1s->1s
- narrow assertion windows widened: honcho first-turn wait 0.4..0.65 ->
  0.25..2.0 (property is bounded-not-hung, not an exact wall-clock);
  compression fork-lock TTL 1s->3s (12 refresh chances per lease);
  compression-lock expiry margins symmetric (ttl 0.05->0.5, sleep 1.0)
- telegram hung-DNS bound 1.0->1.4 (fake hang is 1.5s — must stay under)
This commit is contained in:
Teknium 2026-07-17 06:58:52 -07:00
parent 6929d13941
commit 0f56e20d23
No known key found for this signature in database
31 changed files with 167 additions and 117 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/agent/test_cascading_interrupt_6600.py
View file

@ -76,7 +76,7 @@ def test_non_streaming_cancel_does_not_surface_network_error():
# The forced RemoteProtocolError must NOT surface as the raised error.
assert create_calls["n"] == 1
assert elapsed < 3.0, f"interrupt took {elapsed:.1f}s — should be near-instant"
assert elapsed < 10.0, f"interrupt took {elapsed:.1f}s — should be near-instant (guarding the 30s+ hang)"
def test_normal_transient_error_still_raises_when_not_cancelled():

9
tests/agent/test_compression_concurrent_fork.py
View file

@ -252,7 +252,10 @@ def test_lock_refresh_keeps_owner_live_past_initial_ttl(tmp_path: Path, monkeypa
db.create_session(parent_sid, source="discord")
agent_a = _build_agent_with_db(db, parent_sid)
agent_a._compression_lock_ttl_seconds = 1.0
# 3s TTL / 0.25s refresh: ~12 refresh opportunities per lease. A 1s TTL
# left one missed scheduling quantum between "refreshed" and "expired"
# on a loaded runner.
agent_a._compression_lock_ttl_seconds = 3.0
agent_a._compression_lock_refresh_interval = 0.25
compression_started = threading.Event()
release_compression = threading.Event()
@ -276,9 +279,9 @@ def test_lock_refresh_keeps_owner_live_past_initial_ttl(tmp_path: Path, monkeypa
try:
assert compression_started.wait(timeout=10), "compression never acquired its lock"
assert db.get_compression_lock_holder(parent_sid) is not None
time.sleep(1.2)
time.sleep(3.5)
assert db.try_acquire_compression_lock(
parent_sid, "refresh_probe", ttl_seconds=1.0
parent_sid, "refresh_probe", ttl_seconds=3.0
) is False, "live owner lease expired and was reclaimable before compression finished"
finally:
release_compression.set()

8
tests/agent/test_thread_scoped_output.py
View file

@ -60,8 +60,9 @@ def test_concurrent_thread_keeps_output_during_silence_window():
t2 = threading.Thread(target=loud_worker)
t1.start()
t2.start()
t1.join(timeout=3.0)
t2.join(timeout=3.0)
t1.join(timeout=15.0)
t2.join(timeout=15.0)
assert not t1.is_alive() and not t2.is_alive(), "worker threads didn't finish"
captured = _run_with_real_stream(body)
assert "SILENCED" not in captured
@ -139,7 +140,8 @@ def test_many_concurrent_silenced_and_loud_threads():
t.start()
start.set()
for t in threads:
t.join(timeout=3.0)
t.join(timeout=15.0)
assert not any(t.is_alive() for t in threads), "straggler thread would truncate captured output"
captured = _run_with_real_stream(body)
for i in range(5):

21
tests/agent/test_title_generator.py
View file

@ -375,10 +375,13 @@ class TestMaybeAutoTitle:
]
with patch("agent.title_generator.auto_title_session") as mock_auto:
import threading
called = threading.Event()
mock_auto.side_effect = lambda *a, **k: called.set()
maybe_auto_title(db, "sess-1", "hello", "hi there", history)
# Wait for the daemon thread to complete
import time
time.sleep(0.3)
# Event-based wait: sleep-sync flaked when the daemon thread
# wasn't scheduled within the fixed nap on a loaded runner.
assert called.wait(timeout=10), "auto_title thread never ran"
mock_auto.assert_called_once_with(
db,
"sess-1",
@ -420,9 +423,11 @@ class TestMaybeAutoTitle:
pass
with patch("agent.title_generator.auto_title_session") as mock_auto:
import threading
called = threading.Event()
mock_auto.side_effect = lambda *a, **k: called.set()
maybe_auto_title(db, "sess-1", "hello", "hi there", history, failure_callback=_cb)
import time
time.sleep(0.3)
assert called.wait(timeout=10), "auto_title thread never ran"
mock_auto.assert_called_once_with(
db,
"sess-1",
@ -571,8 +576,10 @@ class TestRuntimeValidator:
return True
with patch("agent.title_generator.auto_title_session") as mock_auto:
import threading
called = threading.Event()
mock_auto.side_effect = lambda *a, **k: called.set()
maybe_auto_title(db, "sess-1", "hello", "hi there", history, runtime_validator=_v)
import time
time.sleep(0.3)
assert called.wait(timeout=10), "auto_title thread never ran"
kwargs = mock_auto.call_args.kwargs
assert kwargs["runtime_validator"] is _v

5
tests/cli/test_cli_interrupt_subagent.py
View file

@ -151,10 +151,11 @@ class TestCLISubagentInterrupt(unittest.TestCase):
print(f"Child {i}._interrupt_requested: {child._interrupt_requested}")
# Wait for child to detect interrupt
detected = interrupt_detected.wait(timeout=3.0)
detected = interrupt_detected.wait(timeout=10.0)
# Wait for delegate to finish
agent_thread.join(timeout=5)
agent_thread.join(timeout=15)
assert not agent_thread.is_alive(), "delegate thread did not finish"
if delegate_error[0]:
raise delegate_error[0]

43
tests/docker/test_profile_gateway.py
View file

@ -66,6 +66,25 @@ def _svstat_wants_up(container: str) -> bool:
return "want up" in state
def _wait_for_want_state(container_name: str, want_up: bool, timeout: float = 15.0) -> None:
"""Poll s6 want-state until it matches, instead of a fixed sleep.
s6 state transitions are asynchronous; fixed two-second sleeps flaked
on loaded CI hosts.
"""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if _svstat_wants_up(container_name) == want_up:
return
time.sleep(0.5)
state = "up" if want_up else "down"
raise AssertionError(
f"slot want-state never became {state} within {timeout}s: "
f"{_svstat(container_name)!r}"
)
def test_profile_create_then_gateway_start(
built_image: str, container_name: str,
) -> None:
@ -88,20 +107,12 @@ def test_profile_create_then_gateway_start(
# supervision-state contract holds. See ``_svstat_wants_up`` for
# why we accept both ``up …`` (currently up) and ``down …, want
# up`` (down but s6 wants up).
time.sleep(2)
assert _svstat_wants_up(container_name), (
f"slot want-state is not up after gateway start: "
f"{_svstat(container_name)!r}"
)
_wait_for_want_state(container_name, want_up=True)
r = _sh(container_name, f"hermes -p {PROFILE} gateway stop", timeout=30)
assert r.returncode == 0
time.sleep(2)
assert not _svstat_wants_up(container_name), (
f"slot want-state still up after gateway stop: "
f"{_svstat(container_name)!r}"
)
_wait_for_want_state(container_name, want_up=False)
def test_profile_delete_stops_gateway(
@ -113,7 +124,7 @@ def test_profile_delete_stops_gateway(
_sh(container_name, f"hermes profile create {PROFILE}")
_sh(container_name, f"hermes -p {PROFILE} gateway start", timeout=60)
time.sleep(3)
_wait_for_want_state(container_name, want_up=True)
r = _sh(
container_name,
@ -122,7 +133,11 @@ def test_profile_delete_stops_gateway(
)
assert r.returncode == 0, f"profile delete failed: {r.stderr}"
time.sleep(2)
# Service slot should be gone.
r = _sh(container_name, f"test -d /run/service/gateway-{PROFILE}")
# Poll for slot removal instead of a fixed sleep.
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
r = _sh(container_name, f"test -d /run/service/gateway-{PROFILE}")
if r.returncode != 0:
break
time.sleep(0.5)
assert r.returncode != 0, "s6 service slot still present after profile delete"

19
tests/docker/test_zombie_reaping.py
View file

@ -29,11 +29,18 @@ def test_orphan_zombies_reaped(
docker_exec_sh(
container_name, "( ( sleep 0.1 & ) & ); sleep 1", timeout=10,
)
time.sleep(1)
r = docker_exec(container_name, "ps", "axo", "stat,pid,comm")
zombies = [
line for line in r.stdout.split("\n")
if line.strip().startswith("Z")
]
# Poll for zombies-absent instead of a fixed sleep: reaping is
# asynchronous (SIGCHLD) and can lag on a loaded host.
deadline = time.monotonic() + 10
zombies = ["(never checked)"]
while time.monotonic() < deadline:
r = docker_exec(container_name, "ps", "axo", "stat,pid,comm")
zombies = [
line for line in r.stdout.split("\n")
if line.strip().startswith("Z")
]
if not zombies:
break
time.sleep(0.5)
assert not zombies, f"Zombies not reaped by PID 1: {zombies}"

20
tests/gateway/test_planned_stop_watcher.py
View file

@ -83,7 +83,7 @@ def test_watcher_fires_shutdown_when_marker_appears(tmp_path, monkeypatch):
daemon=True,
)
watcher.start()
watcher.join(timeout=2.0)
watcher.join(timeout=10.0)
assert not watcher.is_alive(), "Watcher should exit after firing"
assert len(loop._captured) == 1, (
@ -117,7 +117,7 @@ def test_watcher_does_not_fire_when_marker_absent(tmp_path, monkeypatch):
watcher.start()
time.sleep(0.3) # let it poll a few times
stop_event.set()
watcher.join(timeout=2.0)
watcher.join(timeout=10.0)
assert not watcher.is_alive()
assert loop._captured == [], (
@ -153,7 +153,7 @@ def test_watcher_skips_when_runner_already_draining(tmp_path, monkeypatch):
watcher.start()
time.sleep(0.2)
stop_event.set()
watcher.join(timeout=2.0)
watcher.join(timeout=10.0)
assert loop._captured == [], "Watcher fired while runner was already draining"
@ -182,7 +182,7 @@ def test_watcher_skips_when_runner_not_started(tmp_path, monkeypatch):
watcher.start()
time.sleep(0.2)
stop_event.set()
watcher.join(timeout=2.0)
watcher.join(timeout=10.0)
assert loop._captured == [], "Watcher fired before runner was running"
@ -207,11 +207,11 @@ def test_watcher_responds_to_stop_event_promptly(tmp_path, monkeypatch):
time.sleep(0.05)
started_stop = time.monotonic()
stop_event.set()
watcher.join(timeout=2.0)
watcher.join(timeout=10.0)
elapsed = time.monotonic() - started_stop
assert not watcher.is_alive()
assert elapsed < 0.5, f"Watcher took {elapsed:.2f}s to honour stop_event"
assert elapsed < 2.0 # 0.05s-poll thread; loose bound for scheduler stalls, f"Watcher took {elapsed:.2f}s to honour stop_event"
def test_watcher_fires_only_once_when_marker_persists(tmp_path, monkeypatch):
@ -239,7 +239,7 @@ def test_watcher_fires_only_once_when_marker_persists(tmp_path, monkeypatch):
)
watcher.start()
# Let the watcher tick several times — but it should exit after the first fire.
watcher.join(timeout=1.0)
watcher.join(timeout=10.0)
assert not watcher.is_alive()
assert len(loop._captured) == 1, (
@ -276,7 +276,7 @@ def test_watcher_tolerates_marker_path_resolution_errors(tmp_path, monkeypatch,
watcher.start()
time.sleep(0.2)
stop_event.set()
watcher.join(timeout=2.0)
watcher.join(timeout=10.0)
assert not watcher.is_alive(), "Watcher should still honour stop_event after errors"
# No shutdown fired because the marker never reported existence.
@ -326,7 +326,7 @@ def test_watcher_does_not_fire_for_foreign_pid_marker(tmp_path, monkeypatch):
watcher.start()
time.sleep(0.3) # several poll cycles
stop_event.set()
watcher.join(timeout=2.0)
watcher.join(timeout=10.0)
assert not watcher.is_alive()
assert loop._captured == [], (
@ -360,7 +360,7 @@ def test_watcher_cleans_up_stale_marker_and_keeps_running(tmp_path, monkeypatch)
watcher.start()
time.sleep(0.3)
stop_event.set()
watcher.join(timeout=2.0)
watcher.join(timeout=10.0)
assert not watcher.is_alive()
assert loop._captured == [], "Stale marker must not fire shutdown"

28
tests/gateway/test_session_store_lock_io.py
View file

@ -209,16 +209,16 @@ def test_concurrent_same_key_returns_one_published_session(tmp_path):
def synchronized_query(**kwargs):
owner_started.set()
assert release_owner.wait(timeout=2)
assert release_owner.wait(timeout=10)
return original_query(**kwargs)
store._query_recoverable_session = synchronized_query # type: ignore[method-assign]
with ThreadPoolExecutor(max_workers=2) as pool:
owner = pool.submit(store.get_or_create_session, source)
assert owner_started.wait(timeout=2)
assert owner_started.wait(timeout=10)
follower = pool.submit(store.get_or_create_session, source)
release_owner.set()
entries = [owner.result(timeout=2), follower.result(timeout=2)]
entries = [owner.result(timeout=10), follower.result(timeout=10)]
key = store._generate_session_key(source)
assert entries[0] is entries[1]
@ -238,16 +238,16 @@ def test_concurrent_force_new_returns_one_published_session(tmp_path):
def synchronized_impl(*args, **kwargs):
owner_started.set()
assert release_owner.wait(timeout=2)
assert release_owner.wait(timeout=10)
return original_impl(*args, **kwargs)
store._get_or_create_session_impl = synchronized_impl # type: ignore[method-assign]
with ThreadPoolExecutor(max_workers=2) as pool:
owner = pool.submit(store.get_or_create_session, source, True)
assert owner_started.wait(timeout=2)
assert owner_started.wait(timeout=10)
follower = pool.submit(store.get_or_create_session, source, True)
release_owner.set()
entries = [owner.result(timeout=2), follower.result(timeout=2)]
entries = [owner.result(timeout=10), follower.result(timeout=10)]
assert entries[0] is entries[1]
created_ids = {call.kwargs["session_id"] for call in db.create_session.call_args_list}
@ -296,7 +296,7 @@ def test_legacy_and_off_lock_saves_share_one_serialization_lock(tmp_path):
call_number = write_count
if call_number == 1:
first_write_started.set()
assert release_first_write.wait(timeout=2)
assert release_first_write.wait(timeout=10)
persisted = dict(entries)
db.replace_gateway_routing_entries.side_effect = replace
@ -314,12 +314,12 @@ def test_legacy_and_off_lock_saves_share_one_serialization_lock(tmp_path):
with ThreadPoolExecutor(max_workers=2) as pool:
future_a = pool.submit(store._save_entries)
assert first_write_started.wait(timeout=2)
assert first_write_started.wait(timeout=10)
_seed_entry(store, key_b, "sid-b")
future_b = pool.submit(store._save)
release_first_write.set()
future_a.result(timeout=2)
future_b.result(timeout=2)
future_a.result(timeout=10)
future_b.result(timeout=10)
assert set(persisted) == {key_a, key_b}
@ -340,7 +340,7 @@ def test_save_serialization_snapshots_latest_routing_index(tmp_path):
call_number = write_count
if call_number == 1:
first_write_started.set()
assert release_first_write.wait(timeout=2)
assert release_first_write.wait(timeout=10)
persisted = dict(entries)
db.replace_gateway_routing_entries.side_effect = replace
@ -358,12 +358,12 @@ def test_save_serialization_snapshots_latest_routing_index(tmp_path):
with ThreadPoolExecutor(max_workers=2) as pool:
future_a = pool.submit(store._save_entries)
assert first_write_started.wait(timeout=2)
assert first_write_started.wait(timeout=10)
entry_b = _seed_entry(store, key_b, "sid-b")
future_b = pool.submit(store._save_entries)
release_first_write.set()
future_a.result(timeout=2)
future_b.result(timeout=2)
future_a.result(timeout=10)
future_b.result(timeout=10)
assert set(store._entries) == {key_a, key_b}
assert set(persisted) == {key_a, key_b}

4
tests/gateway/test_telegram_network.py
View file

@ -730,7 +730,7 @@ class TestDiscoverFallbackIps:
elapsed = _time.monotonic() - start
assert ips == ["149.154.167.220"]
assert elapsed < 1.0, f"discovery gated on hung system DNS ({elapsed:.2f}s)"
assert elapsed < 1.4, f"discovery gated on hung system DNS ({elapsed:.2f}s)"
@pytest.mark.asyncio
async def test_hung_system_dns_with_no_doh_answers_bounded_seed_fallback(self, monkeypatch):
@ -755,4 +755,4 @@ class TestDiscoverFallbackIps:
elapsed = _time.monotonic() - start
assert ips == tnet._SEED_FALLBACK_IPS
assert elapsed < 1.0, f"seed fallback gated on hung system DNS ({elapsed:.2f}s)"
assert elapsed < 1.4, f"seed fallback gated on hung system DNS ({elapsed:.2f}s)"

4
tests/test_hermes_state_compression_locks.py
View file

@ -84,8 +84,8 @@ def test_locks_are_per_session(db: SessionDB) -> None:
def test_expired_lock_is_reclaimable(db: SessionDB) -> None:
"""A crashed compressor must not permanently block the session."""
# Acquire with a very short TTL
db.try_acquire_compression_lock("sess1", "crashed_holder", ttl_seconds=0.05)
time.sleep(0.1)
db.try_acquire_compression_lock("sess1", "crashed_holder", ttl_seconds=0.5)
time.sleep(1.0)
# Holder check honours expiry
assert db.get_compression_lock_holder("sess1") is None
# New holder can claim it

9
tests/test_honcho_startup_fail_open.py
View file

@ -211,10 +211,15 @@ def test_first_turn_base_wait_is_shared_by_init_and_context_fetch():
started = time.perf_counter()
assert provider.prefetch("what do you know about me?") == ""
elapsed = time.perf_counter() - started
assert 0.4 <= elapsed < 0.65
# Property: prefetch waits for init (0.3s sleep) but is bounded by
# first_turn_base_wait rather than blocking forever on the slow
# context fetch. The old 0.4..0.65 window was 0.25s wide — pure
# scheduler noise on a loaded runner. Lower bound proves the wait
# happened; loose upper bound proves it didn't hang.
assert 0.25 <= elapsed < 2.0
finally:
release_context.set()
provider._init_thread.join(timeout=1)
provider._init_thread.join(timeout=10)

6
tests/tools/test_clarify_gateway.py
View file

@ -40,7 +40,7 @@ class TestClarifyPrimitive:
cm.resolve_gateway_clarify("id1", "B")
threading.Thread(target=resolver).start()
result = cm.wait_for_response("id1", timeout=2.0)
result = cm.wait_for_response("id1", timeout=10.0)
assert result == "B"
def test_open_ended_auto_awaits_text(self):
@ -149,7 +149,7 @@ class TestClarifyPrimitive:
time.sleep(0.05)
cancelled = cm.clear_session("sk7")
assert cancelled == 1
result = fut.result(timeout=2.0)
result = fut.result(timeout=10.0)
# clear_session sets response="" then the wait returns it
assert result == ""
@ -177,7 +177,7 @@ class TestClarifyPrimitive:
cm.unregister_notify("sk9")
# unregister_notify calls clear_session; thread unwinds
result = fut.result(timeout=2.0)
result = fut.result(timeout=10.0)
assert result == ""
def test_session_index_isolation(self):

2
tests/tools/test_docker_environment.py
View file

@ -1199,7 +1199,7 @@ def test_wait_for_cleanup_returns_true_when_no_thread_started():
shutdowns."""
env = docker_env.DockerEnvironment.__new__(docker_env.DockerEnvironment)
# No _cleanup_thread set — simulates an env that was never cleanup()'d.
assert env.wait_for_cleanup(timeout=1.0) is True
assert env.wait_for_cleanup(timeout=10.0) is True
def test_wait_for_cleanup_after_cleanup_returns_true(monkeypatch):

2
tests/tools/test_interrupt.py
View file

@ -50,7 +50,7 @@ class TestInterruptModule:
# Target the checker thread's ident so it sees the interrupt
set_interrupt(True, thread_id=t.ident)
t.join(timeout=1)
t.join(timeout=5)
assert seen["value"]
set_interrupt(False, thread_id=t.ident)

14
tests/tools/test_local_background_child_hang.py
View file

@ -43,7 +43,7 @@ class TestBackgroundChildDoesNotHang:
result = local_env.execute(cmd, timeout=15)
elapsed = time.monotonic() - t0
assert elapsed < 4.0, (
assert elapsed < 10.0, ( # hang under guard is 15s+; loose bound rides out runner stalls
f"terminal_tool hung for {elapsed:.1f}s — drain thread "
f"is still blocking on backgrounded child's inherited pipe fd"
)
@ -63,7 +63,7 @@ class TestBackgroundChildDoesNotHang:
result = local_env.execute(cmd, timeout=15)
elapsed = time.monotonic() - t0
assert elapsed < 4.0, f"setsid+disown path hung for {elapsed:.1f}s"
assert elapsed < 10.0, f"setsid+disown path hung for {elapsed:.1f}s"
assert result["returncode"] == 0
assert "started" in result["output"]
finally:
@ -77,7 +77,7 @@ class TestBackgroundChildDoesNotHang:
elapsed = time.monotonic() - t0
# Loop body sleeps ~0.6s total — elapsed should be close to that.
assert 0.5 < elapsed < 3.0
assert 0.5 < elapsed < 10.0
assert result["returncode"] == 0
for expected in ("tick 1", "tick 2", "tick 3", "done"):
assert expected in result["output"], f"missing {expected!r}"
@ -148,7 +148,7 @@ class TestBackgroundChildDoesNotHang:
result = local_env.execute(command, timeout=1, bounded_capture=True)
elapsed = time.monotonic() - started
assert elapsed < 4.0
assert elapsed < 10.0
assert result["returncode"] == 124
assert len(result["output"]) <= 5_000
assert "[OUTPUT TRUNCATED" in result["output"]
@ -160,13 +160,13 @@ class TestBackgroundChildDoesNotHang:
result = local_env.execute("sleep 30", timeout=2)
elapsed = time.monotonic() - t0
assert elapsed < 4.0
assert elapsed < 10.0
assert result["returncode"] == 124
assert "timed out" in result["output"].lower()
def test_utf8_output_decoded_correctly(self, local_env):
"""Multibyte UTF-8 chunks must decode cleanly under select-based reads."""
result = local_env.execute("echo 日本語 café résumé", timeout=5)
result = local_env.execute("echo 日本語 café résumé", timeout=30)
assert result["returncode"] == 0
assert "日本語" in result["output"]
assert "café" in result["output"]
@ -209,7 +209,7 @@ class TestBackgroundChildDoesNotHang:
'sys.stdout.buffer.write(b"\\xff\\xfe"); '
'sys.stdout.buffer.write(b" after\\n")\''
)
result = local_env.execute(cmd, timeout=5)
result = local_env.execute(cmd, timeout=15)
assert result["returncode"] == 0
assert "before" in result["output"]
assert "after" in result["output"]

8
tests/tools/test_local_interrupt_cleanup.py
View file

@ -49,7 +49,7 @@ def _process_group_snapshot(pgid: int) -> str:
).stdout.strip()
def _wait_for_pgid_exit(pgid: int, timeout: float = 30.0) -> bool:
def _wait_for_pgid_exit(pgid: int, timeout: float = 60.0) -> bool:
"""Wait for a process group to disappear under loaded xdist hosts.
The cleanup chain is: SIGTERM → 3s TimeoutStopSec → SIGKILL → reap.
@ -150,7 +150,7 @@ def test_wait_for_process_kills_subprocess_on_keyboardinterrupt():
# does init_session() (one spawn) before the real command, so we need
# to wait until a sleep 30 is visible. Use pgrep-style lookup via
# /proc to find the bash process running our sleep.
deadline = time.monotonic() + 5.0
deadline = time.monotonic() + 20.0 # generous: init_session + spawn dilate under CI load
target_pid = None
while time.monotonic() < deadline:
# Walk our children and grand-children to find one running 'sleep 30'
@ -201,8 +201,8 @@ def test_wait_for_process_kills_subprocess_on_keyboardinterrupt():
# run the except-block cleanup (_kill_process), and exit. Under
# xdist load the SIGTERM → 3s wait → SIGKILL chain can take longer
# than 5s before the worker's join() returns; bumped to 15s.
t.join(timeout=15.0)
assert not t.is_alive(), "worker didn't exit within 15 s of the interrupt"
t.join(timeout=30.0)
assert not t.is_alive(), "worker didn't exit within 30 s of the interrupt"
# The critical assertion: the subprocess GROUP must be dead. Not
# just the bash wrapper — the 'sleep 30' child too. Under xdist load,

4
tests/tools/test_mcp_cancelled_error_propagation.py
View file

@ -46,7 +46,7 @@ class TestCancelledErrorPropagation:
# CancelledError propagation or clean exit) rather than
# hanging forever.
try:
await asyncio.wait_for(task, timeout=2.0)
await asyncio.wait_for(task, timeout=15.0)
except asyncio.CancelledError:
return "cancelled_cleanly"
except asyncio.TimeoutError:
@ -82,7 +82,7 @@ class TestCancelledErrorPropagation:
server._shutdown_event.set()
server._task.cancel()
try:
await asyncio.wait_for(server._task, timeout=2.0)
await asyncio.wait_for(server._task, timeout=15.0)
except (asyncio.CancelledError, asyncio.TimeoutError):
pass
return server._task.done()

4
tests/tools/test_mcp_circuit_breaker.py
View file

@ -483,7 +483,7 @@ def test_run_loop_parks_instead_of_exiting_then_revives(monkeypatch, tmp_path):
task._shutdown_event.set()
task._reconnect_event.set()
try:
await asyncio.wait_for(run_task, timeout=2)
await asyncio.wait_for(run_task, timeout=15)
except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
run_task.cancel()
@ -564,7 +564,7 @@ def test_initial_connect_budget_parks_instead_of_exiting_then_revives(monkeypatc
task._shutdown_event.set()
task._reconnect_event.set()
try:
await asyncio.wait_for(run_task, timeout=2)
await asyncio.wait_for(run_task, timeout=15)
except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
run_task.cancel()

2
tests/tools/test_mcp_parked_self_probe.py
View file

@ -101,7 +101,7 @@ def test_parked_server_self_probes_and_revives(monkeypatch, tmp_path):
task._shutdown_event.set()
task._reconnect_event.set()
try:
await asyncio.wait_for(run_task, timeout=2)
await asyncio.wait_for(run_task, timeout=15)
except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
run_task.cancel()

4
tests/tools/test_mcp_reconnect_retry_reset.py
View file

@ -120,7 +120,7 @@ def test_reconnect_counter_resets_after_successful_session(monkeypatch, tmp_path
task._shutdown_event.set()
task._reconnect_event.set()
try:
await asyncio.wait_for(run_task, timeout=2)
await asyncio.wait_for(run_task, timeout=15)
except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
run_task.cancel()
@ -189,7 +189,7 @@ def test_reconnect_counter_still_parks_on_consecutive_failures(monkeypatch, tmp_
task._shutdown_event.set()
task._reconnect_event.set()
try:
await asyncio.wait_for(run_task, timeout=2)
await asyncio.wait_for(run_task, timeout=15)
except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
run_task.cancel()

6
tests/tools/test_mcp_stability.py
View file

@ -539,8 +539,8 @@ class TestStdioPgroupReaping:
)
parent_pgid = os.getpgid(parent.pid)
# Wait for parent to exit and grandchild to spin up.
parent.wait(timeout=5)
deadline = _time.time() + 5
parent.wait(timeout=15)
deadline = _time.time() + 15 # fresh CPython spinup dilates under CI load
while _time.time() < deadline and not grandchild_pid_file.exists():
_time.sleep(0.05)
assert grandchild_pid_file.exists(), "grandchild did not start"
@ -577,7 +577,7 @@ class TestStdioPgroupReaping:
pass
# Grandchild should be gone — SIGTERM via killpg in phase 1 reached it.
deadline = _time.time() + 3
deadline = _time.time() + 10
while _time.time() < deadline and psutil.pid_exists(grandchild_pid):
_time.sleep(0.05)
assert not psutil.pid_exists(grandchild_pid), (

6
tests/tools/test_mcp_tool.py
View file

@ -871,7 +871,7 @@ class TestRunOnMCPLoopInterrupts:
try:
with pytest.raises(InterruptedError, match="User sent a new message"):
mcp_mod._run_on_mcp_loop(_slow_call(), timeout=2)
mcp_mod._run_on_mcp_loop(_slow_call(), timeout=10)
deadline = time.time() + 2
while time.time() < deadline and not cancelled.is_set():
@ -880,7 +880,7 @@ class TestRunOnMCPLoopInterrupts:
finally:
set_interrupt(False, waiter_tid)
loop.call_soon_threadsafe(loop.stop)
thread.join(timeout=2)
thread.join(timeout=10)
loop.close()
mcp_mod._mcp_loop = old_loop
mcp_mod._mcp_thread = old_thread
@ -917,7 +917,7 @@ class TestRunOnMCPLoopInterrupts:
assert cancelled.is_set()
finally:
loop.call_soon_threadsafe(loop.stop)
thread.join(timeout=2)
thread.join(timeout=10)
loop.close()
mcp_mod._mcp_loop = old_loop
mcp_mod._mcp_thread = old_thread

13
tests/tools/test_process_registry.py
View file

@ -403,7 +403,7 @@ class TestOrphanedPipeReconciliation:
assert result["status"] == "exited", result
assert result["exit_code"] == 0
assert elapsed < 0.3, f"wait() should wake on completion; took {elapsed:.3f}s"
assert elapsed < 0.9 # must stay under the old 1s poll tick being regression-tested, f"wait() should wake on completion; took {elapsed:.3f}s"
# =========================================================================
@ -2110,8 +2110,11 @@ class TestSigkillEscalation:
sometimes a child). The escalation now re-probes every target directly.
"""
import psutil
# 2.0s grace (not 1.0): with three interpreters mid-startup on a
# loaded runner, a 1s SIGTERM->partition window races child spawn and
# is how a child PID escaped the live-system guard in CI.
monkeypatch.setattr(ProcessRegistry, "_daemon_term_grace_seconds",
staticmethod(lambda: 1.0))
staticmethod(lambda: 2.0))
# Parent spawns 2 children; all trap SIGTERM. Parent prints child pids
# after the handler is installed.
parent_src = (
@ -2125,6 +2128,12 @@ class TestSigkillEscalation:
)
parent = subprocess.Popen([sys.executable, "-c", parent_src],
stdout=subprocess.PIPE, text=True)
# Bound the readline: if the parent wedges before printing, fail THIS
# test with a clear message instead of letting the per-file timeout
# SIGKILL the whole pytest process (opaque rc=124 in CI).
import select as _select
ready, _, _ = _select.select([parent.stdout], [], [], 20.0)
assert ready, "parent process failed to print child pids within 20s"
child_pids = [int(x) for x in parent.stdout.readline().split()]
all_pids = [parent.pid] + child_pids
try:

16
tests/tools/test_registry.py
View file

@ -505,7 +505,7 @@ class TestThreadSafety:
def blocking_check():
check_started.set()
writer_completed_during_check["value"] = writer_done.wait(timeout=1)
writer_completed_during_check["value"] = writer_done.wait(timeout=10)
return True
reg.register(
@ -529,7 +529,7 @@ class TestThreadSafety:
errors.append(exc)
def writer():
assert check_started.wait(timeout=1)
assert check_started.wait(timeout=10)
reg.register(
name="gamma",
toolset="new",
@ -542,8 +542,8 @@ class TestThreadSafety:
writer_thread = threading.Thread(target=writer)
reader_thread.start()
writer_thread.start()
reader_thread.join(timeout=2)
writer_thread.join(timeout=2)
reader_thread.join(timeout=15)
writer_thread.join(timeout=15)
assert not reader_thread.is_alive()
assert not writer_thread.is_alive()
@ -565,7 +565,7 @@ class TestThreadSafety:
def blocking_check():
check_started.set()
writer_completed_during_check["value"] = writer_done.wait(timeout=1)
writer_completed_during_check["value"] = writer_done.wait(timeout=10)
return True
reg.register(
@ -589,7 +589,7 @@ class TestThreadSafety:
errors.append(exc)
def writer():
assert check_started.wait(timeout=1)
assert check_started.wait(timeout=10)
reg.deregister("beta")
writer_done.set()
@ -597,8 +597,8 @@ class TestThreadSafety:
writer_thread = threading.Thread(target=writer)
reader_thread.start()
writer_thread.start()
reader_thread.join(timeout=2)
writer_thread.join(timeout=2)
reader_thread.join(timeout=15)
writer_thread.join(timeout=15)
assert not reader_thread.is_alive()
assert not writer_thread.is_alive()

3
tests/tools/test_voice_cli_integration.py
View file

@ -1322,6 +1322,7 @@ class TestRefreshLevelLock:
with lock:
recording = False
t.join(timeout=1)
t.join(timeout=10)
assert not t.is_alive()
assert not t.is_alive(), "Refresh thread did not stop"
assert iterations > 0, "Refresh thread never ran"

2
tests/tools/test_voice_mode.py
View file

@ -738,7 +738,7 @@ class TestAudioRecorderProperties:
# Force start time to 1 second ago
recorder._start_time = time.monotonic() - 1.0
elapsed = recorder.elapsed_seconds
assert 0.9 < elapsed < 2.0
assert 0.9 < elapsed < 10.0 # loose upper bound; only the lower bound is the property
recorder.cancel()

4
tests/tui_gateway/test_inline_rpc_gil_starvation.py
View file

@ -109,7 +109,7 @@ def test_dispatch_inline_rpc_does_not_block_under_gil_pressure(server):
fast_elapsed = time.monotonic() - t0
assert fast_resp["result"] == {"ok": True}
assert fast_elapsed < 0.5, (
assert fast_elapsed < 2.0, (
f"fast handler blocked for {fast_elapsed:.2f}s behind slow session.list — "
f"the WS read loop would stall, causing false 'needs setup' (#50005)."
)
@ -140,7 +140,7 @@ def test_dispatch_pet_info_does_not_block_prompt_submit(server):
elapsed = time.monotonic() - t0
assert resp["result"] == {"status": "streaming"}
assert elapsed < 0.5, (
assert elapsed < 2.0, (
f"prompt.submit blocked for {elapsed:.2f}s behind slow pet.info — "
f"the user's message would appear stuck under GIL pressure (#50005)."
)

4
tests/tui_gateway/test_iso_certify_seam.py
View file

@ -56,7 +56,7 @@ def test_synth_turn_holds_duration_and_streams():
result = agent.run_conversation(spec, stream_callback=deltas.append)
elapsed = time.monotonic() - t0
# Held for ~the requested wall time (allow generous upper bound under load).
assert 0.35 <= elapsed <= 1.5, elapsed
assert 0.35 <= elapsed <= 5.0, elapsed
assert result["interrupted"] is False
assert len(deltas) >= 3
# Token accounting advanced (the 100K-token heavy-turn proxy).
@ -79,7 +79,7 @@ def test_synth_turn_interrupt_aborts_promptly():
result = agent.run_conversation('{"duration_s": 10.0}')
elapsed = time.monotonic() - t0
assert result["interrupted"] is True
assert elapsed < 1.5, elapsed
assert elapsed < 5.0, elapsed
def test_synth_turn_non_json_prompt_uses_defaults(monkeypatch):

6
tests/tui_gateway/test_protocol.py
View file

@ -1852,7 +1852,7 @@ def test_dispatch_long_handler_does_not_block_fast_handler(server):
fast_elapsed = time.monotonic() - t0
assert fast_resp["result"] == {"pong": True}
assert fast_elapsed < 0.5, f"fast handler blocked for {fast_elapsed:.2f}s behind slow handler"
assert fast_elapsed < 2.0, f"fast handler blocked for {fast_elapsed:.2f}s behind slow handler"
released.set()
@ -1875,7 +1875,7 @@ def test_dispatch_session_compress_does_not_block_fast_handler(server):
fast_elapsed = time.monotonic() - t0
assert fast_resp["result"] == {"pong": True}
assert fast_elapsed < 0.5, f"fast handler blocked for {fast_elapsed:.2f}s behind session.compress"
assert fast_elapsed < 2.0, f"fast handler blocked for {fast_elapsed:.2f}s behind session.compress"
released.set()
@ -1940,6 +1940,6 @@ def test_slow_completion_does_not_block_fast_handler(completion_method, server):
fast_elapsed = time.monotonic() - t0
assert fast_resp["result"] == {"pong": True}
assert fast_elapsed < 0.5, f"fast handler blocked for {fast_elapsed:.2f}s behind {completion_method}"
assert fast_elapsed < 2.0, f"fast handler blocked for {fast_elapsed:.2f}s behind {completion_method}"
released.set()

6
tests/tui_gateway/test_wait_for_mcp_discovery.py
View file

@ -25,7 +25,7 @@ def test_no_thread_is_noop():
entry._mcp_discovery_thread = None
start = time.monotonic()
entry.wait_for_mcp_discovery(timeout=5.0)
assert time.monotonic() - start < 0.1
assert time.monotonic() - start < 1.0 # fast path; loose bound for loaded runners
finally:
_restore_thread_slot(saved)
@ -40,7 +40,7 @@ def test_already_finished_thread_is_noop():
entry._mcp_discovery_thread = t
start = time.monotonic()
entry.wait_for_mcp_discovery(timeout=5.0)
assert time.monotonic() - start < 0.1
assert time.monotonic() - start < 1.0 # fast path; loose bound for loaded runners
finally:
_restore_thread_slot(saved)
@ -71,7 +71,7 @@ def test_hung_thread_is_bounded_by_timeout():
start = time.monotonic()
entry.wait_for_mcp_discovery(timeout=0.3)
elapsed = time.monotonic() - start
assert 0.25 <= elapsed < 1.0 # bounded near the timeout, not forever
assert 0.25 <= elapsed < 3.0 # bounded near the timeout, not forever
assert t.is_alive() # thread still running; we did not block on it
finally:
stop.set()