mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(tests): harden env isolation and replace remaining sleep-sync races
The full 42k-test run and complete npm check surfaced three more classes: - Environment isolation: local ~/.honcho defaultHost and SSH_* variables leaked into Python/TUI tests. Pin the default Honcho host in the hermetic fixture, isolate the one fallback test from ~/.honcho, and blank SSH_* around terminalSetup tests. This flipped 20 false failures back to deterministic behavior on developer machines. - Background-thread sleep-sync: Honcho async writer tests patched time.sleep globally, then busy-polled with that same mocked sleep. Under full-suite load the poller could starve the writer. Each test now waits on an Event emitted by the exact flush/retry transition; 30/30 passed under 15-way contention. - Desktop streaming: the test slept 80ms and assumed a 500ms timer could not fire before its assertion. A loaded runner descheduled the test for >500ms and both chunks arrived. Producer controls now gate second-chunk and completion transitions explicitly. Also make file-retry observability complete: a self-healed flaky file now prints BOTH attempts' full output in the FLAKY summary. Two behavioral runner tests prove pass-on-retry is green+loud+traceback-preserving, while a deterministic failure remains red.
This commit is contained in:
parent
a27c8c94d9
commit
ca115aac0b
7 changed files with 177 additions and 41 deletions
|
|
@ -277,3 +277,85 @@ def test_positional_path_not_treated_as_flag(tmp_path: Path) -> None:
|
|||
# Discovery found the probe file (2 tests), proving the positional path
|
||||
# was consumed as a root, not forwarded to pytest as a bad flag.
|
||||
assert "test_flagprobe.py" in proc.stdout, proc.stdout
|
||||
|
||||
|
||||
def test_file_retry_self_heals_and_prints_both_attempts(tmp_path: Path) -> None:
|
||||
"""A pass-on-retry is green, loud, and retains the failing traceback."""
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
runner = repo_root / "scripts" / "run_tests_parallel.py"
|
||||
marker = tmp_path / "ran-once"
|
||||
probe = tmp_path / "test_flaky_probe.py"
|
||||
probe.write_text(
|
||||
textwrap.dedent(
|
||||
f"""
|
||||
from pathlib import Path
|
||||
|
||||
def test_flaky_once():
|
||||
marker = Path({str(marker)!r})
|
||||
if not marker.exists():
|
||||
marker.write_text("failed once")
|
||||
assert False, "simulated first-attempt flake"
|
||||
assert True
|
||||
"""
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(runner),
|
||||
"--files",
|
||||
str(probe),
|
||||
"--file-retries",
|
||||
"1",
|
||||
"-j",
|
||||
"1",
|
||||
"-q",
|
||||
],
|
||||
cwd=repo_root,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert proc.returncode == 0, proc.stdout
|
||||
assert "FLAKY file" in proc.stdout
|
||||
assert "simulated first-attempt flake" in proc.stdout
|
||||
assert "first-attempt output" in proc.stdout
|
||||
assert "retry output" in proc.stdout
|
||||
|
||||
|
||||
def test_file_retry_does_not_launder_deterministic_failure(tmp_path: Path) -> None:
|
||||
"""A real regression fails both attempts and the runner remains red."""
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
runner = repo_root / "scripts" / "run_tests_parallel.py"
|
||||
probe = tmp_path / "test_red_probe.py"
|
||||
probe.write_text(
|
||||
"def test_always_red():\n assert False, 'deterministic regression'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(runner),
|
||||
"--files",
|
||||
str(probe),
|
||||
"--file-retries",
|
||||
"1",
|
||||
"-j",
|
||||
"1",
|
||||
"-q",
|
||||
],
|
||||
cwd=repo_root,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert proc.returncode == 1, proc.stdout
|
||||
assert "deterministic regression" in proc.stdout
|
||||
assert "FLAKY file" not in proc.stdout
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue