From 0f56e20d2391f85d2d315bc24c0e621bf60074ae Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Fri, 17 Jul 2026 06:58:52 -0700 Subject: [PATCH] test: de-flake 30 timing-sensitive test files for loaded CI runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root-cause fixes from the flake audit (session-DB mining + repo sweep): Event-based sync instead of sleep-sync: - title_generator: mock sets threading.Event, wait(10) replaces sleep(0.3) hoping the daemon thread got scheduled - docker zombie_reaping / profile_gateway: poll-for-state helpers replace fixed 1-3s sleeps (s6 transitions + SIGCHLD reaping are async) - process_registry tree test: select()-bounded readline replaces an unbounded blocking read (parent wedge now fails THIS test with a clear message instead of an opaque rc=124 file kill); SIGTERM grace 1s->2s (the 1s partition window mid-interpreter-startup is how a child PID escaped the live-system guard in CI) Timeout raises (loaded 8-way-sliced runners see ~5s scheduling floors; all of these complete in ms-to-1s when healthy so the raises cost nothing on green runs): - subprocess/thread waits <= 2s raised to 10-15s across mcp_tool, mcp_circuit_breaker, mcp_reconnect_retry_reset, mcp_parked_self_probe, mcp_cancelled_error_propagation, registry, clarify_gateway, interrupt, voice_cli_integration, docker_environment, session_store_lock_io, planned_stop_watcher, cli_interrupt_subagent, thread_scoped_output (joins now also assert not is_alive() so stragglers fail loudly) - wall-clock discrimination ceilings loosened where the guarded hang is 10x larger: local_background_child_hang 4s->10s, interrupt_cleanup setup 5s->20s + pgid-exit 30s->60s, mcp_stability grandchild spinup 5s->15s, protocol/gil-starvation fast-handler 0.5s->2s, iso_certify_seam 1.5s->5s, wait_for_mcp_discovery 0.1s->1s - narrow assertion windows widened: honcho first-turn wait 0.4..0.65 -> 0.25..2.0 (property is bounded-not-hung, not an exact wall-clock); compression fork-lock TTL 1s->3s (12 refresh chances per lease); compression-lock expiry margins symmetric (ttl 0.05->0.5, sleep 1.0) - telegram hung-DNS bound 1.0->1.4 (fake hang is 1.5s — must stay under) --- tests/agent/test_cascading_interrupt_6600.py | 2 +- .../agent/test_compression_concurrent_fork.py | 9 ++-- tests/agent/test_thread_scoped_output.py | 8 ++-- tests/agent/test_title_generator.py | 21 ++++++--- tests/cli/test_cli_interrupt_subagent.py | 5 ++- tests/docker/test_profile_gateway.py | 43 +++++++++++++------ tests/docker/test_zombie_reaping.py | 19 +++++--- tests/gateway/test_planned_stop_watcher.py | 20 ++++----- tests/gateway/test_session_store_lock_io.py | 28 ++++++------ tests/gateway/test_telegram_network.py | 4 +- tests/test_hermes_state_compression_locks.py | 4 +- tests/test_honcho_startup_fail_open.py | 9 +++- tests/tools/test_clarify_gateway.py | 6 +-- tests/tools/test_docker_environment.py | 2 +- tests/tools/test_interrupt.py | 2 +- .../tools/test_local_background_child_hang.py | 14 +++--- tests/tools/test_local_interrupt_cleanup.py | 8 ++-- .../test_mcp_cancelled_error_propagation.py | 4 +- tests/tools/test_mcp_circuit_breaker.py | 4 +- tests/tools/test_mcp_parked_self_probe.py | 2 +- tests/tools/test_mcp_reconnect_retry_reset.py | 4 +- tests/tools/test_mcp_stability.py | 6 +-- tests/tools/test_mcp_tool.py | 6 +-- tests/tools/test_process_registry.py | 13 +++++- tests/tools/test_registry.py | 16 +++---- tests/tools/test_voice_cli_integration.py | 3 +- tests/tools/test_voice_mode.py | 2 +- .../test_inline_rpc_gil_starvation.py | 4 +- tests/tui_gateway/test_iso_certify_seam.py | 4 +- tests/tui_gateway/test_protocol.py | 6 +-- .../test_wait_for_mcp_discovery.py | 6 +-- 31 files changed, 167 insertions(+), 117 deletions(-) diff --git a/tests/agent/test_cascading_interrupt_6600.py b/tests/agent/test_cascading_interrupt_6600.py index 58fc28c4df0..b335c6a6b17 100644 --- a/tests/agent/test_cascading_interrupt_6600.py +++ b/tests/agent/test_cascading_interrupt_6600.py @@ -76,7 +76,7 @@ def test_non_streaming_cancel_does_not_surface_network_error(): # The forced RemoteProtocolError must NOT surface as the raised error. assert create_calls["n"] == 1 - assert elapsed < 3.0, f"interrupt took {elapsed:.1f}s — should be near-instant" + assert elapsed < 10.0, f"interrupt took {elapsed:.1f}s — should be near-instant (guarding the 30s+ hang)" def test_normal_transient_error_still_raises_when_not_cancelled(): diff --git a/tests/agent/test_compression_concurrent_fork.py b/tests/agent/test_compression_concurrent_fork.py index b23ad37da1e..51870c5ffea 100644 --- a/tests/agent/test_compression_concurrent_fork.py +++ b/tests/agent/test_compression_concurrent_fork.py @@ -252,7 +252,10 @@ def test_lock_refresh_keeps_owner_live_past_initial_ttl(tmp_path: Path, monkeypa db.create_session(parent_sid, source="discord") agent_a = _build_agent_with_db(db, parent_sid) - agent_a._compression_lock_ttl_seconds = 1.0 + # 3s TTL / 0.25s refresh: ~12 refresh opportunities per lease. A 1s TTL + # left one missed scheduling quantum between "refreshed" and "expired" + # on a loaded runner. + agent_a._compression_lock_ttl_seconds = 3.0 agent_a._compression_lock_refresh_interval = 0.25 compression_started = threading.Event() release_compression = threading.Event() @@ -276,9 +279,9 @@ def test_lock_refresh_keeps_owner_live_past_initial_ttl(tmp_path: Path, monkeypa try: assert compression_started.wait(timeout=10), "compression never acquired its lock" assert db.get_compression_lock_holder(parent_sid) is not None - time.sleep(1.2) + time.sleep(3.5) assert db.try_acquire_compression_lock( - parent_sid, "refresh_probe", ttl_seconds=1.0 + parent_sid, "refresh_probe", ttl_seconds=3.0 ) is False, "live owner lease expired and was reclaimable before compression finished" finally: release_compression.set() diff --git a/tests/agent/test_thread_scoped_output.py b/tests/agent/test_thread_scoped_output.py index 3899905463f..d697e7a2a5d 100644 --- a/tests/agent/test_thread_scoped_output.py +++ b/tests/agent/test_thread_scoped_output.py @@ -60,8 +60,9 @@ def test_concurrent_thread_keeps_output_during_silence_window(): t2 = threading.Thread(target=loud_worker) t1.start() t2.start() - t1.join(timeout=3.0) - t2.join(timeout=3.0) + t1.join(timeout=15.0) + t2.join(timeout=15.0) + assert not t1.is_alive() and not t2.is_alive(), "worker threads didn't finish" captured = _run_with_real_stream(body) assert "SILENCED" not in captured @@ -139,7 +140,8 @@ def test_many_concurrent_silenced_and_loud_threads(): t.start() start.set() for t in threads: - t.join(timeout=3.0) + t.join(timeout=15.0) + assert not any(t.is_alive() for t in threads), "straggler thread would truncate captured output" captured = _run_with_real_stream(body) for i in range(5): diff --git a/tests/agent/test_title_generator.py b/tests/agent/test_title_generator.py index 321f9030ae0..bd281f728e0 100644 --- a/tests/agent/test_title_generator.py +++ b/tests/agent/test_title_generator.py @@ -375,10 +375,13 @@ class TestMaybeAutoTitle: ] with patch("agent.title_generator.auto_title_session") as mock_auto: + import threading + called = threading.Event() + mock_auto.side_effect = lambda *a, **k: called.set() maybe_auto_title(db, "sess-1", "hello", "hi there", history) - # Wait for the daemon thread to complete - import time - time.sleep(0.3) + # Event-based wait: sleep-sync flaked when the daemon thread + # wasn't scheduled within the fixed nap on a loaded runner. + assert called.wait(timeout=10), "auto_title thread never ran" mock_auto.assert_called_once_with( db, "sess-1", @@ -420,9 +423,11 @@ class TestMaybeAutoTitle: pass with patch("agent.title_generator.auto_title_session") as mock_auto: + import threading + called = threading.Event() + mock_auto.side_effect = lambda *a, **k: called.set() maybe_auto_title(db, "sess-1", "hello", "hi there", history, failure_callback=_cb) - import time - time.sleep(0.3) + assert called.wait(timeout=10), "auto_title thread never ran" mock_auto.assert_called_once_with( db, "sess-1", @@ -571,8 +576,10 @@ class TestRuntimeValidator: return True with patch("agent.title_generator.auto_title_session") as mock_auto: + import threading + called = threading.Event() + mock_auto.side_effect = lambda *a, **k: called.set() maybe_auto_title(db, "sess-1", "hello", "hi there", history, runtime_validator=_v) - import time - time.sleep(0.3) + assert called.wait(timeout=10), "auto_title thread never ran" kwargs = mock_auto.call_args.kwargs assert kwargs["runtime_validator"] is _v diff --git a/tests/cli/test_cli_interrupt_subagent.py b/tests/cli/test_cli_interrupt_subagent.py index 5b732425c8a..7088bf30b76 100644 --- a/tests/cli/test_cli_interrupt_subagent.py +++ b/tests/cli/test_cli_interrupt_subagent.py @@ -151,10 +151,11 @@ class TestCLISubagentInterrupt(unittest.TestCase): print(f"Child {i}._interrupt_requested: {child._interrupt_requested}") # Wait for child to detect interrupt - detected = interrupt_detected.wait(timeout=3.0) + detected = interrupt_detected.wait(timeout=10.0) # Wait for delegate to finish - agent_thread.join(timeout=5) + agent_thread.join(timeout=15) + assert not agent_thread.is_alive(), "delegate thread did not finish" if delegate_error[0]: raise delegate_error[0] diff --git a/tests/docker/test_profile_gateway.py b/tests/docker/test_profile_gateway.py index fdf017bbceb..2adadf7e377 100644 --- a/tests/docker/test_profile_gateway.py +++ b/tests/docker/test_profile_gateway.py @@ -66,6 +66,25 @@ def _svstat_wants_up(container: str) -> bool: return "want up" in state + +def _wait_for_want_state(container_name: str, want_up: bool, timeout: float = 15.0) -> None: + """Poll s6 want-state until it matches, instead of a fixed sleep. + + s6 state transitions are asynchronous; fixed two-second sleeps flaked + on loaded CI hosts. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if _svstat_wants_up(container_name) == want_up: + return + time.sleep(0.5) + state = "up" if want_up else "down" + raise AssertionError( + f"slot want-state never became {state} within {timeout}s: " + f"{_svstat(container_name)!r}" + ) + + def test_profile_create_then_gateway_start( built_image: str, container_name: str, ) -> None: @@ -88,20 +107,12 @@ def test_profile_create_then_gateway_start( # supervision-state contract holds. See ``_svstat_wants_up`` for # why we accept both ``up …`` (currently up) and ``down …, want # up`` (down but s6 wants up). - time.sleep(2) - assert _svstat_wants_up(container_name), ( - f"slot want-state is not up after gateway start: " - f"{_svstat(container_name)!r}" - ) + _wait_for_want_state(container_name, want_up=True) r = _sh(container_name, f"hermes -p {PROFILE} gateway stop", timeout=30) assert r.returncode == 0 - time.sleep(2) - assert not _svstat_wants_up(container_name), ( - f"slot want-state still up after gateway stop: " - f"{_svstat(container_name)!r}" - ) + _wait_for_want_state(container_name, want_up=False) def test_profile_delete_stops_gateway( @@ -113,7 +124,7 @@ def test_profile_delete_stops_gateway( _sh(container_name, f"hermes profile create {PROFILE}") _sh(container_name, f"hermes -p {PROFILE} gateway start", timeout=60) - time.sleep(3) + _wait_for_want_state(container_name, want_up=True) r = _sh( container_name, @@ -122,7 +133,11 @@ def test_profile_delete_stops_gateway( ) assert r.returncode == 0, f"profile delete failed: {r.stderr}" - time.sleep(2) - # Service slot should be gone. - r = _sh(container_name, f"test -d /run/service/gateway-{PROFILE}") + # Poll for slot removal instead of a fixed sleep. + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + r = _sh(container_name, f"test -d /run/service/gateway-{PROFILE}") + if r.returncode != 0: + break + time.sleep(0.5) assert r.returncode != 0, "s6 service slot still present after profile delete" \ No newline at end of file diff --git a/tests/docker/test_zombie_reaping.py b/tests/docker/test_zombie_reaping.py index dc14d3c93ca..e136a510891 100644 --- a/tests/docker/test_zombie_reaping.py +++ b/tests/docker/test_zombie_reaping.py @@ -29,11 +29,18 @@ def test_orphan_zombies_reaped( docker_exec_sh( container_name, "( ( sleep 0.1 & ) & ); sleep 1", timeout=10, ) - time.sleep(1) - r = docker_exec(container_name, "ps", "axo", "stat,pid,comm") - zombies = [ - line for line in r.stdout.split("\n") - if line.strip().startswith("Z") - ] + # Poll for zombies-absent instead of a fixed sleep: reaping is + # asynchronous (SIGCHLD) and can lag on a loaded host. + deadline = time.monotonic() + 10 + zombies = ["(never checked)"] + while time.monotonic() < deadline: + r = docker_exec(container_name, "ps", "axo", "stat,pid,comm") + zombies = [ + line for line in r.stdout.split("\n") + if line.strip().startswith("Z") + ] + if not zombies: + break + time.sleep(0.5) assert not zombies, f"Zombies not reaped by PID 1: {zombies}" \ No newline at end of file diff --git a/tests/gateway/test_planned_stop_watcher.py b/tests/gateway/test_planned_stop_watcher.py index 451a3d8f8a7..47c801248f0 100644 --- a/tests/gateway/test_planned_stop_watcher.py +++ b/tests/gateway/test_planned_stop_watcher.py @@ -83,7 +83,7 @@ def test_watcher_fires_shutdown_when_marker_appears(tmp_path, monkeypatch): daemon=True, ) watcher.start() - watcher.join(timeout=2.0) + watcher.join(timeout=10.0) assert not watcher.is_alive(), "Watcher should exit after firing" assert len(loop._captured) == 1, ( @@ -117,7 +117,7 @@ def test_watcher_does_not_fire_when_marker_absent(tmp_path, monkeypatch): watcher.start() time.sleep(0.3) # let it poll a few times stop_event.set() - watcher.join(timeout=2.0) + watcher.join(timeout=10.0) assert not watcher.is_alive() assert loop._captured == [], ( @@ -153,7 +153,7 @@ def test_watcher_skips_when_runner_already_draining(tmp_path, monkeypatch): watcher.start() time.sleep(0.2) stop_event.set() - watcher.join(timeout=2.0) + watcher.join(timeout=10.0) assert loop._captured == [], "Watcher fired while runner was already draining" @@ -182,7 +182,7 @@ def test_watcher_skips_when_runner_not_started(tmp_path, monkeypatch): watcher.start() time.sleep(0.2) stop_event.set() - watcher.join(timeout=2.0) + watcher.join(timeout=10.0) assert loop._captured == [], "Watcher fired before runner was running" @@ -207,11 +207,11 @@ def test_watcher_responds_to_stop_event_promptly(tmp_path, monkeypatch): time.sleep(0.05) started_stop = time.monotonic() stop_event.set() - watcher.join(timeout=2.0) + watcher.join(timeout=10.0) elapsed = time.monotonic() - started_stop assert not watcher.is_alive() - assert elapsed < 0.5, f"Watcher took {elapsed:.2f}s to honour stop_event" + assert elapsed < 2.0 # 0.05s-poll thread; loose bound for scheduler stalls, f"Watcher took {elapsed:.2f}s to honour stop_event" def test_watcher_fires_only_once_when_marker_persists(tmp_path, monkeypatch): @@ -239,7 +239,7 @@ def test_watcher_fires_only_once_when_marker_persists(tmp_path, monkeypatch): ) watcher.start() # Let the watcher tick several times — but it should exit after the first fire. - watcher.join(timeout=1.0) + watcher.join(timeout=10.0) assert not watcher.is_alive() assert len(loop._captured) == 1, ( @@ -276,7 +276,7 @@ def test_watcher_tolerates_marker_path_resolution_errors(tmp_path, monkeypatch, watcher.start() time.sleep(0.2) stop_event.set() - watcher.join(timeout=2.0) + watcher.join(timeout=10.0) assert not watcher.is_alive(), "Watcher should still honour stop_event after errors" # No shutdown fired because the marker never reported existence. @@ -326,7 +326,7 @@ def test_watcher_does_not_fire_for_foreign_pid_marker(tmp_path, monkeypatch): watcher.start() time.sleep(0.3) # several poll cycles stop_event.set() - watcher.join(timeout=2.0) + watcher.join(timeout=10.0) assert not watcher.is_alive() assert loop._captured == [], ( @@ -360,7 +360,7 @@ def test_watcher_cleans_up_stale_marker_and_keeps_running(tmp_path, monkeypatch) watcher.start() time.sleep(0.3) stop_event.set() - watcher.join(timeout=2.0) + watcher.join(timeout=10.0) assert not watcher.is_alive() assert loop._captured == [], "Stale marker must not fire shutdown" diff --git a/tests/gateway/test_session_store_lock_io.py b/tests/gateway/test_session_store_lock_io.py index a1d1fd1e885..8c82a036a7a 100644 --- a/tests/gateway/test_session_store_lock_io.py +++ b/tests/gateway/test_session_store_lock_io.py @@ -209,16 +209,16 @@ def test_concurrent_same_key_returns_one_published_session(tmp_path): def synchronized_query(**kwargs): owner_started.set() - assert release_owner.wait(timeout=2) + assert release_owner.wait(timeout=10) return original_query(**kwargs) store._query_recoverable_session = synchronized_query # type: ignore[method-assign] with ThreadPoolExecutor(max_workers=2) as pool: owner = pool.submit(store.get_or_create_session, source) - assert owner_started.wait(timeout=2) + assert owner_started.wait(timeout=10) follower = pool.submit(store.get_or_create_session, source) release_owner.set() - entries = [owner.result(timeout=2), follower.result(timeout=2)] + entries = [owner.result(timeout=10), follower.result(timeout=10)] key = store._generate_session_key(source) assert entries[0] is entries[1] @@ -238,16 +238,16 @@ def test_concurrent_force_new_returns_one_published_session(tmp_path): def synchronized_impl(*args, **kwargs): owner_started.set() - assert release_owner.wait(timeout=2) + assert release_owner.wait(timeout=10) return original_impl(*args, **kwargs) store._get_or_create_session_impl = synchronized_impl # type: ignore[method-assign] with ThreadPoolExecutor(max_workers=2) as pool: owner = pool.submit(store.get_or_create_session, source, True) - assert owner_started.wait(timeout=2) + assert owner_started.wait(timeout=10) follower = pool.submit(store.get_or_create_session, source, True) release_owner.set() - entries = [owner.result(timeout=2), follower.result(timeout=2)] + entries = [owner.result(timeout=10), follower.result(timeout=10)] assert entries[0] is entries[1] created_ids = {call.kwargs["session_id"] for call in db.create_session.call_args_list} @@ -296,7 +296,7 @@ def test_legacy_and_off_lock_saves_share_one_serialization_lock(tmp_path): call_number = write_count if call_number == 1: first_write_started.set() - assert release_first_write.wait(timeout=2) + assert release_first_write.wait(timeout=10) persisted = dict(entries) db.replace_gateway_routing_entries.side_effect = replace @@ -314,12 +314,12 @@ def test_legacy_and_off_lock_saves_share_one_serialization_lock(tmp_path): with ThreadPoolExecutor(max_workers=2) as pool: future_a = pool.submit(store._save_entries) - assert first_write_started.wait(timeout=2) + assert first_write_started.wait(timeout=10) _seed_entry(store, key_b, "sid-b") future_b = pool.submit(store._save) release_first_write.set() - future_a.result(timeout=2) - future_b.result(timeout=2) + future_a.result(timeout=10) + future_b.result(timeout=10) assert set(persisted) == {key_a, key_b} @@ -340,7 +340,7 @@ def test_save_serialization_snapshots_latest_routing_index(tmp_path): call_number = write_count if call_number == 1: first_write_started.set() - assert release_first_write.wait(timeout=2) + assert release_first_write.wait(timeout=10) persisted = dict(entries) db.replace_gateway_routing_entries.side_effect = replace @@ -358,12 +358,12 @@ def test_save_serialization_snapshots_latest_routing_index(tmp_path): with ThreadPoolExecutor(max_workers=2) as pool: future_a = pool.submit(store._save_entries) - assert first_write_started.wait(timeout=2) + assert first_write_started.wait(timeout=10) entry_b = _seed_entry(store, key_b, "sid-b") future_b = pool.submit(store._save_entries) release_first_write.set() - future_a.result(timeout=2) - future_b.result(timeout=2) + future_a.result(timeout=10) + future_b.result(timeout=10) assert set(store._entries) == {key_a, key_b} assert set(persisted) == {key_a, key_b} diff --git a/tests/gateway/test_telegram_network.py b/tests/gateway/test_telegram_network.py index 8406973b948..1615200dff3 100644 --- a/tests/gateway/test_telegram_network.py +++ b/tests/gateway/test_telegram_network.py @@ -730,7 +730,7 @@ class TestDiscoverFallbackIps: elapsed = _time.monotonic() - start assert ips == ["149.154.167.220"] - assert elapsed < 1.0, f"discovery gated on hung system DNS ({elapsed:.2f}s)" + assert elapsed < 1.4, f"discovery gated on hung system DNS ({elapsed:.2f}s)" @pytest.mark.asyncio async def test_hung_system_dns_with_no_doh_answers_bounded_seed_fallback(self, monkeypatch): @@ -755,4 +755,4 @@ class TestDiscoverFallbackIps: elapsed = _time.monotonic() - start assert ips == tnet._SEED_FALLBACK_IPS - assert elapsed < 1.0, f"seed fallback gated on hung system DNS ({elapsed:.2f}s)" + assert elapsed < 1.4, f"seed fallback gated on hung system DNS ({elapsed:.2f}s)" diff --git a/tests/test_hermes_state_compression_locks.py b/tests/test_hermes_state_compression_locks.py index 53e3bc0dec4..4e44d92a445 100644 --- a/tests/test_hermes_state_compression_locks.py +++ b/tests/test_hermes_state_compression_locks.py @@ -84,8 +84,8 @@ def test_locks_are_per_session(db: SessionDB) -> None: def test_expired_lock_is_reclaimable(db: SessionDB) -> None: """A crashed compressor must not permanently block the session.""" # Acquire with a very short TTL - db.try_acquire_compression_lock("sess1", "crashed_holder", ttl_seconds=0.05) - time.sleep(0.1) + db.try_acquire_compression_lock("sess1", "crashed_holder", ttl_seconds=0.5) + time.sleep(1.0) # Holder check honours expiry assert db.get_compression_lock_holder("sess1") is None # New holder can claim it diff --git a/tests/test_honcho_startup_fail_open.py b/tests/test_honcho_startup_fail_open.py index 2570a7a6c73..0ed3be796ee 100644 --- a/tests/test_honcho_startup_fail_open.py +++ b/tests/test_honcho_startup_fail_open.py @@ -211,10 +211,15 @@ def test_first_turn_base_wait_is_shared_by_init_and_context_fetch(): started = time.perf_counter() assert provider.prefetch("what do you know about me?") == "" elapsed = time.perf_counter() - started - assert 0.4 <= elapsed < 0.65 + # Property: prefetch waits for init (0.3s sleep) but is bounded by + # first_turn_base_wait rather than blocking forever on the slow + # context fetch. The old 0.4..0.65 window was 0.25s wide — pure + # scheduler noise on a loaded runner. Lower bound proves the wait + # happened; loose upper bound proves it didn't hang. + assert 0.25 <= elapsed < 2.0 finally: release_context.set() - provider._init_thread.join(timeout=1) + provider._init_thread.join(timeout=10) diff --git a/tests/tools/test_clarify_gateway.py b/tests/tools/test_clarify_gateway.py index bd44ecdb961..c617a4be694 100644 --- a/tests/tools/test_clarify_gateway.py +++ b/tests/tools/test_clarify_gateway.py @@ -40,7 +40,7 @@ class TestClarifyPrimitive: cm.resolve_gateway_clarify("id1", "B") threading.Thread(target=resolver).start() - result = cm.wait_for_response("id1", timeout=2.0) + result = cm.wait_for_response("id1", timeout=10.0) assert result == "B" def test_open_ended_auto_awaits_text(self): @@ -149,7 +149,7 @@ class TestClarifyPrimitive: time.sleep(0.05) cancelled = cm.clear_session("sk7") assert cancelled == 1 - result = fut.result(timeout=2.0) + result = fut.result(timeout=10.0) # clear_session sets response="" then the wait returns it assert result == "" @@ -177,7 +177,7 @@ class TestClarifyPrimitive: cm.unregister_notify("sk9") # unregister_notify calls clear_session; thread unwinds - result = fut.result(timeout=2.0) + result = fut.result(timeout=10.0) assert result == "" def test_session_index_isolation(self): diff --git a/tests/tools/test_docker_environment.py b/tests/tools/test_docker_environment.py index c85402a3c2d..1f89c1aad9f 100644 --- a/tests/tools/test_docker_environment.py +++ b/tests/tools/test_docker_environment.py @@ -1199,7 +1199,7 @@ def test_wait_for_cleanup_returns_true_when_no_thread_started(): shutdowns.""" env = docker_env.DockerEnvironment.__new__(docker_env.DockerEnvironment) # No _cleanup_thread set — simulates an env that was never cleanup()'d. - assert env.wait_for_cleanup(timeout=1.0) is True + assert env.wait_for_cleanup(timeout=10.0) is True def test_wait_for_cleanup_after_cleanup_returns_true(monkeypatch): diff --git a/tests/tools/test_interrupt.py b/tests/tools/test_interrupt.py index 8c71f10ffd9..aca47df9e19 100644 --- a/tests/tools/test_interrupt.py +++ b/tests/tools/test_interrupt.py @@ -50,7 +50,7 @@ class TestInterruptModule: # Target the checker thread's ident so it sees the interrupt set_interrupt(True, thread_id=t.ident) - t.join(timeout=1) + t.join(timeout=5) assert seen["value"] set_interrupt(False, thread_id=t.ident) diff --git a/tests/tools/test_local_background_child_hang.py b/tests/tools/test_local_background_child_hang.py index dd9f112a567..805b96585af 100644 --- a/tests/tools/test_local_background_child_hang.py +++ b/tests/tools/test_local_background_child_hang.py @@ -43,7 +43,7 @@ class TestBackgroundChildDoesNotHang: result = local_env.execute(cmd, timeout=15) elapsed = time.monotonic() - t0 - assert elapsed < 4.0, ( + assert elapsed < 10.0, ( # hang under guard is 15s+; loose bound rides out runner stalls f"terminal_tool hung for {elapsed:.1f}s — drain thread " f"is still blocking on backgrounded child's inherited pipe fd" ) @@ -63,7 +63,7 @@ class TestBackgroundChildDoesNotHang: result = local_env.execute(cmd, timeout=15) elapsed = time.monotonic() - t0 - assert elapsed < 4.0, f"setsid+disown path hung for {elapsed:.1f}s" + assert elapsed < 10.0, f"setsid+disown path hung for {elapsed:.1f}s" assert result["returncode"] == 0 assert "started" in result["output"] finally: @@ -77,7 +77,7 @@ class TestBackgroundChildDoesNotHang: elapsed = time.monotonic() - t0 # Loop body sleeps ~0.6s total — elapsed should be close to that. - assert 0.5 < elapsed < 3.0 + assert 0.5 < elapsed < 10.0 assert result["returncode"] == 0 for expected in ("tick 1", "tick 2", "tick 3", "done"): assert expected in result["output"], f"missing {expected!r}" @@ -148,7 +148,7 @@ class TestBackgroundChildDoesNotHang: result = local_env.execute(command, timeout=1, bounded_capture=True) elapsed = time.monotonic() - started - assert elapsed < 4.0 + assert elapsed < 10.0 assert result["returncode"] == 124 assert len(result["output"]) <= 5_000 assert "[OUTPUT TRUNCATED" in result["output"] @@ -160,13 +160,13 @@ class TestBackgroundChildDoesNotHang: result = local_env.execute("sleep 30", timeout=2) elapsed = time.monotonic() - t0 - assert elapsed < 4.0 + assert elapsed < 10.0 assert result["returncode"] == 124 assert "timed out" in result["output"].lower() def test_utf8_output_decoded_correctly(self, local_env): """Multibyte UTF-8 chunks must decode cleanly under select-based reads.""" - result = local_env.execute("echo 日本語 café résumé", timeout=5) + result = local_env.execute("echo 日本語 café résumé", timeout=30) assert result["returncode"] == 0 assert "日本語" in result["output"] assert "café" in result["output"] @@ -209,7 +209,7 @@ class TestBackgroundChildDoesNotHang: 'sys.stdout.buffer.write(b"\\xff\\xfe"); ' 'sys.stdout.buffer.write(b" after\\n")\'' ) - result = local_env.execute(cmd, timeout=5) + result = local_env.execute(cmd, timeout=15) assert result["returncode"] == 0 assert "before" in result["output"] assert "after" in result["output"] diff --git a/tests/tools/test_local_interrupt_cleanup.py b/tests/tools/test_local_interrupt_cleanup.py index 29f3d466784..364223b9698 100644 --- a/tests/tools/test_local_interrupt_cleanup.py +++ b/tests/tools/test_local_interrupt_cleanup.py @@ -49,7 +49,7 @@ def _process_group_snapshot(pgid: int) -> str: ).stdout.strip() -def _wait_for_pgid_exit(pgid: int, timeout: float = 30.0) -> bool: +def _wait_for_pgid_exit(pgid: int, timeout: float = 60.0) -> bool: """Wait for a process group to disappear under loaded xdist hosts. The cleanup chain is: SIGTERM → 3s TimeoutStopSec → SIGKILL → reap. @@ -150,7 +150,7 @@ def test_wait_for_process_kills_subprocess_on_keyboardinterrupt(): # does init_session() (one spawn) before the real command, so we need # to wait until a sleep 30 is visible. Use pgrep-style lookup via # /proc to find the bash process running our sleep. - deadline = time.monotonic() + 5.0 + deadline = time.monotonic() + 20.0 # generous: init_session + spawn dilate under CI load target_pid = None while time.monotonic() < deadline: # Walk our children and grand-children to find one running 'sleep 30' @@ -201,8 +201,8 @@ def test_wait_for_process_kills_subprocess_on_keyboardinterrupt(): # run the except-block cleanup (_kill_process), and exit. Under # xdist load the SIGTERM → 3s wait → SIGKILL chain can take longer # than 5s before the worker's join() returns; bumped to 15s. - t.join(timeout=15.0) - assert not t.is_alive(), "worker didn't exit within 15 s of the interrupt" + t.join(timeout=30.0) + assert not t.is_alive(), "worker didn't exit within 30 s of the interrupt" # The critical assertion: the subprocess GROUP must be dead. Not # just the bash wrapper — the 'sleep 30' child too. Under xdist load, diff --git a/tests/tools/test_mcp_cancelled_error_propagation.py b/tests/tools/test_mcp_cancelled_error_propagation.py index 13636c3caac..88f232fc254 100644 --- a/tests/tools/test_mcp_cancelled_error_propagation.py +++ b/tests/tools/test_mcp_cancelled_error_propagation.py @@ -46,7 +46,7 @@ class TestCancelledErrorPropagation: # CancelledError propagation or clean exit) rather than # hanging forever. try: - await asyncio.wait_for(task, timeout=2.0) + await asyncio.wait_for(task, timeout=15.0) except asyncio.CancelledError: return "cancelled_cleanly" except asyncio.TimeoutError: @@ -82,7 +82,7 @@ class TestCancelledErrorPropagation: server._shutdown_event.set() server._task.cancel() try: - await asyncio.wait_for(server._task, timeout=2.0) + await asyncio.wait_for(server._task, timeout=15.0) except (asyncio.CancelledError, asyncio.TimeoutError): pass return server._task.done() diff --git a/tests/tools/test_mcp_circuit_breaker.py b/tests/tools/test_mcp_circuit_breaker.py index f356c94e128..357589d0662 100644 --- a/tests/tools/test_mcp_circuit_breaker.py +++ b/tests/tools/test_mcp_circuit_breaker.py @@ -483,7 +483,7 @@ def test_run_loop_parks_instead_of_exiting_then_revives(monkeypatch, tmp_path): task._shutdown_event.set() task._reconnect_event.set() try: - await asyncio.wait_for(run_task, timeout=2) + await asyncio.wait_for(run_task, timeout=15) except (asyncio.TimeoutError, asyncio.CancelledError, Exception): run_task.cancel() @@ -564,7 +564,7 @@ def test_initial_connect_budget_parks_instead_of_exiting_then_revives(monkeypatc task._shutdown_event.set() task._reconnect_event.set() try: - await asyncio.wait_for(run_task, timeout=2) + await asyncio.wait_for(run_task, timeout=15) except (asyncio.TimeoutError, asyncio.CancelledError, Exception): run_task.cancel() diff --git a/tests/tools/test_mcp_parked_self_probe.py b/tests/tools/test_mcp_parked_self_probe.py index 95decf096e0..af53fdea2d4 100644 --- a/tests/tools/test_mcp_parked_self_probe.py +++ b/tests/tools/test_mcp_parked_self_probe.py @@ -101,7 +101,7 @@ def test_parked_server_self_probes_and_revives(monkeypatch, tmp_path): task._shutdown_event.set() task._reconnect_event.set() try: - await asyncio.wait_for(run_task, timeout=2) + await asyncio.wait_for(run_task, timeout=15) except (asyncio.TimeoutError, asyncio.CancelledError, Exception): run_task.cancel() diff --git a/tests/tools/test_mcp_reconnect_retry_reset.py b/tests/tools/test_mcp_reconnect_retry_reset.py index 50442aa602a..0bef5ca7f5c 100644 --- a/tests/tools/test_mcp_reconnect_retry_reset.py +++ b/tests/tools/test_mcp_reconnect_retry_reset.py @@ -120,7 +120,7 @@ def test_reconnect_counter_resets_after_successful_session(monkeypatch, tmp_path task._shutdown_event.set() task._reconnect_event.set() try: - await asyncio.wait_for(run_task, timeout=2) + await asyncio.wait_for(run_task, timeout=15) except (asyncio.TimeoutError, asyncio.CancelledError, Exception): run_task.cancel() @@ -189,7 +189,7 @@ def test_reconnect_counter_still_parks_on_consecutive_failures(monkeypatch, tmp_ task._shutdown_event.set() task._reconnect_event.set() try: - await asyncio.wait_for(run_task, timeout=2) + await asyncio.wait_for(run_task, timeout=15) except (asyncio.TimeoutError, asyncio.CancelledError, Exception): run_task.cancel() diff --git a/tests/tools/test_mcp_stability.py b/tests/tools/test_mcp_stability.py index b9a1b92c9cd..f204615aca0 100644 --- a/tests/tools/test_mcp_stability.py +++ b/tests/tools/test_mcp_stability.py @@ -539,8 +539,8 @@ class TestStdioPgroupReaping: ) parent_pgid = os.getpgid(parent.pid) # Wait for parent to exit and grandchild to spin up. - parent.wait(timeout=5) - deadline = _time.time() + 5 + parent.wait(timeout=15) + deadline = _time.time() + 15 # fresh CPython spinup dilates under CI load while _time.time() < deadline and not grandchild_pid_file.exists(): _time.sleep(0.05) assert grandchild_pid_file.exists(), "grandchild did not start" @@ -577,7 +577,7 @@ class TestStdioPgroupReaping: pass # Grandchild should be gone — SIGTERM via killpg in phase 1 reached it. - deadline = _time.time() + 3 + deadline = _time.time() + 10 while _time.time() < deadline and psutil.pid_exists(grandchild_pid): _time.sleep(0.05) assert not psutil.pid_exists(grandchild_pid), ( diff --git a/tests/tools/test_mcp_tool.py b/tests/tools/test_mcp_tool.py index be41088e4eb..c2208165fa4 100644 --- a/tests/tools/test_mcp_tool.py +++ b/tests/tools/test_mcp_tool.py @@ -871,7 +871,7 @@ class TestRunOnMCPLoopInterrupts: try: with pytest.raises(InterruptedError, match="User sent a new message"): - mcp_mod._run_on_mcp_loop(_slow_call(), timeout=2) + mcp_mod._run_on_mcp_loop(_slow_call(), timeout=10) deadline = time.time() + 2 while time.time() < deadline and not cancelled.is_set(): @@ -880,7 +880,7 @@ class TestRunOnMCPLoopInterrupts: finally: set_interrupt(False, waiter_tid) loop.call_soon_threadsafe(loop.stop) - thread.join(timeout=2) + thread.join(timeout=10) loop.close() mcp_mod._mcp_loop = old_loop mcp_mod._mcp_thread = old_thread @@ -917,7 +917,7 @@ class TestRunOnMCPLoopInterrupts: assert cancelled.is_set() finally: loop.call_soon_threadsafe(loop.stop) - thread.join(timeout=2) + thread.join(timeout=10) loop.close() mcp_mod._mcp_loop = old_loop mcp_mod._mcp_thread = old_thread diff --git a/tests/tools/test_process_registry.py b/tests/tools/test_process_registry.py index 817eaabcc8e..beeb41fe1ea 100644 --- a/tests/tools/test_process_registry.py +++ b/tests/tools/test_process_registry.py @@ -403,7 +403,7 @@ class TestOrphanedPipeReconciliation: assert result["status"] == "exited", result assert result["exit_code"] == 0 - assert elapsed < 0.3, f"wait() should wake on completion; took {elapsed:.3f}s" + assert elapsed < 0.9 # must stay under the old 1s poll tick being regression-tested, f"wait() should wake on completion; took {elapsed:.3f}s" # ========================================================================= @@ -2110,8 +2110,11 @@ class TestSigkillEscalation: sometimes a child). The escalation now re-probes every target directly. """ import psutil + # 2.0s grace (not 1.0): with three interpreters mid-startup on a + # loaded runner, a 1s SIGTERM->partition window races child spawn and + # is how a child PID escaped the live-system guard in CI. monkeypatch.setattr(ProcessRegistry, "_daemon_term_grace_seconds", - staticmethod(lambda: 1.0)) + staticmethod(lambda: 2.0)) # Parent spawns 2 children; all trap SIGTERM. Parent prints child pids # after the handler is installed. parent_src = ( @@ -2125,6 +2128,12 @@ class TestSigkillEscalation: ) parent = subprocess.Popen([sys.executable, "-c", parent_src], stdout=subprocess.PIPE, text=True) + # Bound the readline: if the parent wedges before printing, fail THIS + # test with a clear message instead of letting the per-file timeout + # SIGKILL the whole pytest process (opaque rc=124 in CI). + import select as _select + ready, _, _ = _select.select([parent.stdout], [], [], 20.0) + assert ready, "parent process failed to print child pids within 20s" child_pids = [int(x) for x in parent.stdout.readline().split()] all_pids = [parent.pid] + child_pids try: diff --git a/tests/tools/test_registry.py b/tests/tools/test_registry.py index 49d1193cf67..97415c991e2 100644 --- a/tests/tools/test_registry.py +++ b/tests/tools/test_registry.py @@ -505,7 +505,7 @@ class TestThreadSafety: def blocking_check(): check_started.set() - writer_completed_during_check["value"] = writer_done.wait(timeout=1) + writer_completed_during_check["value"] = writer_done.wait(timeout=10) return True reg.register( @@ -529,7 +529,7 @@ class TestThreadSafety: errors.append(exc) def writer(): - assert check_started.wait(timeout=1) + assert check_started.wait(timeout=10) reg.register( name="gamma", toolset="new", @@ -542,8 +542,8 @@ class TestThreadSafety: writer_thread = threading.Thread(target=writer) reader_thread.start() writer_thread.start() - reader_thread.join(timeout=2) - writer_thread.join(timeout=2) + reader_thread.join(timeout=15) + writer_thread.join(timeout=15) assert not reader_thread.is_alive() assert not writer_thread.is_alive() @@ -565,7 +565,7 @@ class TestThreadSafety: def blocking_check(): check_started.set() - writer_completed_during_check["value"] = writer_done.wait(timeout=1) + writer_completed_during_check["value"] = writer_done.wait(timeout=10) return True reg.register( @@ -589,7 +589,7 @@ class TestThreadSafety: errors.append(exc) def writer(): - assert check_started.wait(timeout=1) + assert check_started.wait(timeout=10) reg.deregister("beta") writer_done.set() @@ -597,8 +597,8 @@ class TestThreadSafety: writer_thread = threading.Thread(target=writer) reader_thread.start() writer_thread.start() - reader_thread.join(timeout=2) - writer_thread.join(timeout=2) + reader_thread.join(timeout=15) + writer_thread.join(timeout=15) assert not reader_thread.is_alive() assert not writer_thread.is_alive() diff --git a/tests/tools/test_voice_cli_integration.py b/tests/tools/test_voice_cli_integration.py index dc6f2061f2c..bf96ff50128 100644 --- a/tests/tools/test_voice_cli_integration.py +++ b/tests/tools/test_voice_cli_integration.py @@ -1322,6 +1322,7 @@ class TestRefreshLevelLock: with lock: recording = False - t.join(timeout=1) + t.join(timeout=10) + assert not t.is_alive() assert not t.is_alive(), "Refresh thread did not stop" assert iterations > 0, "Refresh thread never ran" diff --git a/tests/tools/test_voice_mode.py b/tests/tools/test_voice_mode.py index 8f6a8e67784..b092250e2b4 100644 --- a/tests/tools/test_voice_mode.py +++ b/tests/tools/test_voice_mode.py @@ -738,7 +738,7 @@ class TestAudioRecorderProperties: # Force start time to 1 second ago recorder._start_time = time.monotonic() - 1.0 elapsed = recorder.elapsed_seconds - assert 0.9 < elapsed < 2.0 + assert 0.9 < elapsed < 10.0 # loose upper bound; only the lower bound is the property recorder.cancel() diff --git a/tests/tui_gateway/test_inline_rpc_gil_starvation.py b/tests/tui_gateway/test_inline_rpc_gil_starvation.py index 80244b71a73..99c63c74628 100644 --- a/tests/tui_gateway/test_inline_rpc_gil_starvation.py +++ b/tests/tui_gateway/test_inline_rpc_gil_starvation.py @@ -109,7 +109,7 @@ def test_dispatch_inline_rpc_does_not_block_under_gil_pressure(server): fast_elapsed = time.monotonic() - t0 assert fast_resp["result"] == {"ok": True} - assert fast_elapsed < 0.5, ( + assert fast_elapsed < 2.0, ( f"fast handler blocked for {fast_elapsed:.2f}s behind slow session.list — " f"the WS read loop would stall, causing false 'needs setup' (#50005)." ) @@ -140,7 +140,7 @@ def test_dispatch_pet_info_does_not_block_prompt_submit(server): elapsed = time.monotonic() - t0 assert resp["result"] == {"status": "streaming"} - assert elapsed < 0.5, ( + assert elapsed < 2.0, ( f"prompt.submit blocked for {elapsed:.2f}s behind slow pet.info — " f"the user's message would appear stuck under GIL pressure (#50005)." ) diff --git a/tests/tui_gateway/test_iso_certify_seam.py b/tests/tui_gateway/test_iso_certify_seam.py index 3a9a41342a4..14618d38c27 100644 --- a/tests/tui_gateway/test_iso_certify_seam.py +++ b/tests/tui_gateway/test_iso_certify_seam.py @@ -56,7 +56,7 @@ def test_synth_turn_holds_duration_and_streams(): result = agent.run_conversation(spec, stream_callback=deltas.append) elapsed = time.monotonic() - t0 # Held for ~the requested wall time (allow generous upper bound under load). - assert 0.35 <= elapsed <= 1.5, elapsed + assert 0.35 <= elapsed <= 5.0, elapsed assert result["interrupted"] is False assert len(deltas) >= 3 # Token accounting advanced (the 100K-token heavy-turn proxy). @@ -79,7 +79,7 @@ def test_synth_turn_interrupt_aborts_promptly(): result = agent.run_conversation('{"duration_s": 10.0}') elapsed = time.monotonic() - t0 assert result["interrupted"] is True - assert elapsed < 1.5, elapsed + assert elapsed < 5.0, elapsed def test_synth_turn_non_json_prompt_uses_defaults(monkeypatch): diff --git a/tests/tui_gateway/test_protocol.py b/tests/tui_gateway/test_protocol.py index 5aa7c7d8425..49177c7694e 100644 --- a/tests/tui_gateway/test_protocol.py +++ b/tests/tui_gateway/test_protocol.py @@ -1852,7 +1852,7 @@ def test_dispatch_long_handler_does_not_block_fast_handler(server): fast_elapsed = time.monotonic() - t0 assert fast_resp["result"] == {"pong": True} - assert fast_elapsed < 0.5, f"fast handler blocked for {fast_elapsed:.2f}s behind slow handler" + assert fast_elapsed < 2.0, f"fast handler blocked for {fast_elapsed:.2f}s behind slow handler" released.set() @@ -1875,7 +1875,7 @@ def test_dispatch_session_compress_does_not_block_fast_handler(server): fast_elapsed = time.monotonic() - t0 assert fast_resp["result"] == {"pong": True} - assert fast_elapsed < 0.5, f"fast handler blocked for {fast_elapsed:.2f}s behind session.compress" + assert fast_elapsed < 2.0, f"fast handler blocked for {fast_elapsed:.2f}s behind session.compress" released.set() @@ -1940,6 +1940,6 @@ def test_slow_completion_does_not_block_fast_handler(completion_method, server): fast_elapsed = time.monotonic() - t0 assert fast_resp["result"] == {"pong": True} - assert fast_elapsed < 0.5, f"fast handler blocked for {fast_elapsed:.2f}s behind {completion_method}" + assert fast_elapsed < 2.0, f"fast handler blocked for {fast_elapsed:.2f}s behind {completion_method}" released.set() diff --git a/tests/tui_gateway/test_wait_for_mcp_discovery.py b/tests/tui_gateway/test_wait_for_mcp_discovery.py index ab5bb5f6ddc..d4b3eacf5ac 100644 --- a/tests/tui_gateway/test_wait_for_mcp_discovery.py +++ b/tests/tui_gateway/test_wait_for_mcp_discovery.py @@ -25,7 +25,7 @@ def test_no_thread_is_noop(): entry._mcp_discovery_thread = None start = time.monotonic() entry.wait_for_mcp_discovery(timeout=5.0) - assert time.monotonic() - start < 0.1 + assert time.monotonic() - start < 1.0 # fast path; loose bound for loaded runners finally: _restore_thread_slot(saved) @@ -40,7 +40,7 @@ def test_already_finished_thread_is_noop(): entry._mcp_discovery_thread = t start = time.monotonic() entry.wait_for_mcp_discovery(timeout=5.0) - assert time.monotonic() - start < 0.1 + assert time.monotonic() - start < 1.0 # fast path; loose bound for loaded runners finally: _restore_thread_slot(saved) @@ -71,7 +71,7 @@ def test_hung_thread_is_bounded_by_timeout(): start = time.monotonic() entry.wait_for_mcp_discovery(timeout=0.3) elapsed = time.monotonic() - start - assert 0.25 <= elapsed < 1.0 # bounded near the timeout, not forever + assert 0.25 <= elapsed < 3.0 # bounded near the timeout, not forever assert t.is_alive() # thread still running; we did not block on it finally: stop.set()