fix(cron): resolve SessionDB timeout from config.yaml

Salvage of #63935. The original fix read HERMES_CRON_SESSION_DB_TIMEOUT
from a bare env var, but AGENTS.md requires non-secret behavioral
settings to live in config.yaml with an env var bridge only for
backward compatibility.

Changes:
- Add cron.session_db_timeout_seconds to DEFAULT_CONFIG (default 10s)
- Resolution order: HERMES_CRON_SESSION_DB_TIMEOUT env override →
  cron.session_db_timeout_seconds in config.yaml → 10s default
  (mirrors the existing script_timeout_seconds pattern)
- 0 = unlimited (opt-in for debugging, skips the bound)
- Strengthen test: assert the warning is logged on invalid env value
  (caplog was taken but never asserted)
- Add test: verify config.yaml resolution path works end-to-end

Co-authored-by: LoicHmh <26006141+LoicHmh@users.noreply.github.com>
This commit is contained in:
kshitijk4poor 2026-07-14 03:15:37 +05:30 committed by kshitij
parent c675e7c793
commit ccb045ba7d
3 changed files with 99 additions and 17 deletions

View file

@ -2631,23 +2631,47 @@ def run_job(
_session_db = None
try:
from hermes_state import SessionDB
_raw_session_db_timeout = os.getenv("HERMES_CRON_SESSION_DB_TIMEOUT", "").strip()
try:
_session_db_timeout = float(_raw_session_db_timeout) if _raw_session_db_timeout else 10.0
except (ValueError, TypeError):
logger.warning(
"Invalid HERMES_CRON_SESSION_DB_TIMEOUT=%r; using default 10s",
_raw_session_db_timeout,
)
# Resolve timeout: env override → config.yaml → default 10s.
# Mirrors the script_timeout_seconds resolution pattern.
_session_db_timeout: float | None = None
_raw_env_timeout = os.getenv("HERMES_CRON_SESSION_DB_TIMEOUT", "").strip()
if _raw_env_timeout:
try:
_session_db_timeout = float(_raw_env_timeout)
except (ValueError, TypeError):
logger.warning(
"Invalid HERMES_CRON_SESSION_DB_TIMEOUT=%r; using config/default",
_raw_env_timeout,
)
if _session_db_timeout is None:
try:
from hermes_cli.config import load_config
_cfg = load_config() or {}
_cron_cfg = _cfg.get("cron", {}) if isinstance(_cfg, dict) else {}
_configured = _cron_cfg.get("session_db_timeout_seconds")
if _configured is not None:
_session_db_timeout = float(_configured)
except Exception as exc:
logger.debug(
"Failed to load cron.session_db_timeout_seconds from config: %s",
exc,
)
if _session_db_timeout is None:
_session_db_timeout = 10.0
_session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
try:
_session_db = _session_db_pool.submit(SessionDB).result(timeout=_session_db_timeout)
finally:
# Don't wait for a wedged connect() to unwind — abandon the
# worker thread (same pattern as the agent inactivity timeout
# further down) rather than blocking shutdown on it too.
_session_db_pool.shutdown(wait=False)
if _session_db_timeout > 0:
_session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
try:
_session_db = _session_db_pool.submit(SessionDB).result(timeout=_session_db_timeout)
finally:
# Don't wait for a wedged connect() to unwind — abandon the
# worker thread (same pattern as the agent inactivity timeout
# further down) rather than blocking shutdown on it too.
_session_db_pool.shutdown(wait=False)
else:
# 0 = unlimited (legacy behavior, opt-in for debugging)
_session_db = SessionDB()
except concurrent.futures.TimeoutError:
logger.error(
"Job '%s': SessionDB init did not return within %.0fs — proceeding "

View file

@ -2705,6 +2705,12 @@ DEFAULT_CONFIG = {
# recent .md files and prunes older ones. 0 or negative disables
# pruning (for operators who manage cleanup externally). Default 50.
"output_retention": 50,
# Timeout (seconds) for SessionDB() init inside cron jobs.
# SessionDB opens/migrates state.db synchronously and has no timeout
# of its own against a wedged sqlite3.connect. An unbounded hang here
# wedges the job's dispatch guard forever. Also overridable via
# HERMES_CRON_SESSION_DB_TIMEOUT env var. 0 = unlimited (skip the bound).
"session_db_timeout_seconds": 10,
},
# Kanban multi-agent coordination — controls the dispatcher loop that

View file

@ -107,11 +107,63 @@ class TestSessionDbInitTimeout:
mock_agent.run_conversation.return_value = {"final_response": "ok"}
mock_agent_cls.return_value = mock_agent
success, output, final_response, error = run_job(job)
with caplog.at_level("WARNING"):
success, output, final_response, error = run_job(job)
assert success is True
kwargs = mock_agent_cls.call_args.kwargs
assert kwargs["session_db"] is fake_db # default 10s was plenty for a MagicMock
# The malformed env var must produce a warning so the misconfiguration
# is observable — otherwise it silently falls back and operators can't
# diagnose why their custom timeout isn't taking effect.
assert any(
"HERMES_CRON_SESSION_DB_TIMEOUT" in rec.message
for rec in caplog.records
), f"Expected warning about invalid timeout env var; got: {[r.message for r in caplog.records]}"
def test_timeout_resolved_from_config_yaml(self, tmp_path, monkeypatch):
"""cron.session_db_timeout_seconds in config.yaml is respected when
the env var is not set the canonical config-first resolution path."""
import yaml
monkeypatch.delenv("HERMES_CRON_SESSION_DB_TIMEOUT", raising=False)
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"cron": {"session_db_timeout_seconds": 0.2}})
)
never_set = threading.Event()
job = {"id": "config-timeout", "name": "test", "prompt": "hello"}
try:
with patch("cron.scheduler._hermes_home", tmp_path), \
patch("cron.scheduler._resolve_origin", return_value=None), \
patch("hermes_cli.env_loader.load_hermes_dotenv"), \
patch("hermes_cli.env_loader.reset_secret_source_cache"), \
patch("hermes_state.SessionDB", side_effect=lambda: _hanging_session_db(never_set)), \
patch(
"hermes_cli.runtime_provider.resolve_runtime_provider",
return_value={
"api_key": "test-key",
"base_url": "https://example.invalid/v1",
"provider": "openrouter",
"api_mode": "chat_completions",
},
), \
patch("run_agent.AIAgent") as mock_agent_cls:
mock_agent = MagicMock()
mock_agent.run_conversation.return_value = {"final_response": "ok"}
mock_agent_cls.return_value = mock_agent
start = time.monotonic()
success, output, final_response, error = run_job(job)
elapsed = time.monotonic() - start
finally:
never_set.set()
# Config value 0.2s bounds the hang, not the 10s default.
assert elapsed < 5.0
assert success is True
assert mock_agent_cls.call_args.kwargs["session_db"] is None
class TestDispatchGuardReleasedAfterHang: