mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-15 14:22:43 +00:00
test(cli): deflake --accept-hooks position test — one driver, one import (#61734)
test_accepted_at_every_position spawned 11 separate 'python -m hermes_cli.main' subprocesses, each cold-importing the full CLI module tree under a 15s TimeoutExpired deadline. On a loaded CI worker the import alone can exceed that (slice 2/8 flaked exactly here on PR #61726's run, TimeoutExpired at subprocess.py:1253), failing PRs that never touched the CLI. Replace with ONE driver subprocess that imports hermes_cli.main once and parses all 11 argvs in-process (catching SystemExit per argv), reporting JSON results. Same assertions per argv, identical semantics (verified the --help-before-unknown-flag exit behavior matches the old method), ~11x less import work, and the 180s timeout only trips on a genuine hang.
This commit is contained in:
parent
10c0d9b2a7
commit
b298fd5db1
1 changed files with 48 additions and 11 deletions
|
|
@ -153,7 +153,7 @@ class TestAcceptHooksOnAgentSubparsers:
|
|||
parser and `chat`, so `hermes gateway run --accept-hooks` failed
|
||||
with `unrecognized arguments`."""
|
||||
|
||||
@pytest.mark.parametrize("argv", [
|
||||
ARGVS = [
|
||||
["--accept-hooks", "gateway", "run", "--help"],
|
||||
["gateway", "--accept-hooks", "run", "--help"],
|
||||
["gateway", "run", "--accept-hooks", "--help"],
|
||||
|
|
@ -165,20 +165,57 @@ class TestAcceptHooksOnAgentSubparsers:
|
|||
["mcp", "--accept-hooks", "serve", "--help"],
|
||||
["mcp", "serve", "--accept-hooks", "--help"],
|
||||
["acp", "--accept-hooks", "--help"],
|
||||
])
|
||||
def test_accepted_at_every_position(self, argv):
|
||||
"""Invoking `hermes <argv>` must exit 0 (help) rather than
|
||||
failing with `unrecognized arguments`."""
|
||||
]
|
||||
|
||||
# One driver subprocess parses ALL argvs: hermes_cli.main is a very heavy
|
||||
# import (previously 11 separate `python -m hermes_cli.main` spawns with a
|
||||
# 15s timeout each — a cold import on a loaded CI worker regularly blew
|
||||
# that deadline, making this test flaky). Importing once and parsing 11
|
||||
# times removes the repeated-import cost entirely; the generous timeout
|
||||
# only trips on a genuine hang. `--help` exits via SystemExit(0), which
|
||||
# the driver catches per argv.
|
||||
_DRIVER = r"""
|
||||
import io, json, sys
|
||||
from contextlib import redirect_stdout, redirect_stderr
|
||||
|
||||
import hermes_cli.main as main_mod
|
||||
|
||||
argvs = json.loads(sys.argv[1])
|
||||
results = []
|
||||
for argv in argvs:
|
||||
sys.argv = ["hermes", *argv]
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
code = 0
|
||||
try:
|
||||
with redirect_stdout(out), redirect_stderr(err):
|
||||
main_mod.main()
|
||||
except SystemExit as exc:
|
||||
code = int(exc.code or 0)
|
||||
except Exception as exc: # noqa: BLE001 - report, don't crash the driver
|
||||
code = -1
|
||||
err.write(repr(exc))
|
||||
results.append({"argv": argv, "code": code, "stderr": err.getvalue()[:300]})
|
||||
print(json.dumps(results))
|
||||
"""
|
||||
|
||||
def test_accepted_at_every_position(self):
|
||||
"""Every `hermes <argv>` must exit 0 (help) rather than failing
|
||||
with `unrecognized arguments`."""
|
||||
import json
|
||||
import subprocess
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "hermes_cli.main", *argv],
|
||||
[sys.executable, "-c", self._DRIVER, json.dumps(self.ARGVS)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
timeout=180,
|
||||
)
|
||||
assert result.returncode == 0, (
|
||||
f"argv={argv!r} returned {result.returncode}\n"
|
||||
f"stdout: {result.stdout[:300]}\n"
|
||||
f"stderr: {result.stderr[:300]}"
|
||||
f"driver failed rc={result.returncode}\n"
|
||||
f"stdout: {result.stdout[:500]}\nstderr: {result.stderr[:500]}"
|
||||
)
|
||||
assert "unrecognized arguments" not in result.stderr
|
||||
for entry in json.loads(result.stdout.strip().splitlines()[-1]):
|
||||
assert entry["code"] == 0, (
|
||||
f"argv={entry['argv']!r} returned {entry['code']}\n"
|
||||
f"stderr: {entry['stderr']}"
|
||||
)
|
||||
assert "unrecognized arguments" not in entry["stderr"]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue