From ccb045ba7df3b6796584dd6d1e29bb5327fc838c Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 14 Jul 2026 03:15:37 +0530 Subject: [PATCH] fix(cron): resolve SessionDB timeout from config.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Salvage of #63935. The original fix read HERMES_CRON_SESSION_DB_TIMEOUT from a bare env var, but AGENTS.md requires non-secret behavioral settings to live in config.yaml with an env var bridge only for backward compatibility. Changes: - Add cron.session_db_timeout_seconds to DEFAULT_CONFIG (default 10s) - Resolution order: HERMES_CRON_SESSION_DB_TIMEOUT env override → cron.session_db_timeout_seconds in config.yaml → 10s default (mirrors the existing script_timeout_seconds pattern) - 0 = unlimited (opt-in for debugging, skips the bound) - Strengthen test: assert the warning is logged on invalid env value (caplog was taken but never asserted) - Add test: verify config.yaml resolution path works end-to-end Co-authored-by: LoicHmh <26006141+LoicHmh@users.noreply.github.com> --- cron/scheduler.py | 56 ++++++++++++++++++-------- hermes_cli/config.py | 6 +++ tests/cron/test_sessiondb_init_hang.py | 54 ++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 17 deletions(-) diff --git a/cron/scheduler.py b/cron/scheduler.py index 1a9d4dee5fe4..9d8dc7507757 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -2631,23 +2631,47 @@ def run_job( _session_db = None try: from hermes_state import SessionDB - _raw_session_db_timeout = os.getenv("HERMES_CRON_SESSION_DB_TIMEOUT", "").strip() - try: - _session_db_timeout = float(_raw_session_db_timeout) if _raw_session_db_timeout else 10.0 - except (ValueError, TypeError): - logger.warning( - "Invalid HERMES_CRON_SESSION_DB_TIMEOUT=%r; using default 10s", - _raw_session_db_timeout, - ) + + # Resolve timeout: env override → config.yaml → default 10s. + # Mirrors the script_timeout_seconds resolution pattern. + _session_db_timeout: float | None = None + _raw_env_timeout = os.getenv("HERMES_CRON_SESSION_DB_TIMEOUT", "").strip() + if _raw_env_timeout: + try: + _session_db_timeout = float(_raw_env_timeout) + except (ValueError, TypeError): + logger.warning( + "Invalid HERMES_CRON_SESSION_DB_TIMEOUT=%r; using config/default", + _raw_env_timeout, + ) + if _session_db_timeout is None: + try: + from hermes_cli.config import load_config + _cfg = load_config() or {} + _cron_cfg = _cfg.get("cron", {}) if isinstance(_cfg, dict) else {} + _configured = _cron_cfg.get("session_db_timeout_seconds") + if _configured is not None: + _session_db_timeout = float(_configured) + except Exception as exc: + logger.debug( + "Failed to load cron.session_db_timeout_seconds from config: %s", + exc, + ) + if _session_db_timeout is None: _session_db_timeout = 10.0 - _session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) - try: - _session_db = _session_db_pool.submit(SessionDB).result(timeout=_session_db_timeout) - finally: - # Don't wait for a wedged connect() to unwind — abandon the - # worker thread (same pattern as the agent inactivity timeout - # further down) rather than blocking shutdown on it too. - _session_db_pool.shutdown(wait=False) + + if _session_db_timeout > 0: + _session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) + try: + _session_db = _session_db_pool.submit(SessionDB).result(timeout=_session_db_timeout) + finally: + # Don't wait for a wedged connect() to unwind — abandon the + # worker thread (same pattern as the agent inactivity timeout + # further down) rather than blocking shutdown on it too. + _session_db_pool.shutdown(wait=False) + else: + # 0 = unlimited (legacy behavior, opt-in for debugging) + _session_db = SessionDB() except concurrent.futures.TimeoutError: logger.error( "Job '%s': SessionDB init did not return within %.0fs — proceeding " diff --git a/hermes_cli/config.py b/hermes_cli/config.py index da95d4bdf2b7..b95b1c666a89 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2705,6 +2705,12 @@ DEFAULT_CONFIG = { # recent .md files and prunes older ones. 0 or negative disables # pruning (for operators who manage cleanup externally). Default 50. "output_retention": 50, + # Timeout (seconds) for SessionDB() init inside cron jobs. + # SessionDB opens/migrates state.db synchronously and has no timeout + # of its own against a wedged sqlite3.connect. An unbounded hang here + # wedges the job's dispatch guard forever. Also overridable via + # HERMES_CRON_SESSION_DB_TIMEOUT env var. 0 = unlimited (skip the bound). + "session_db_timeout_seconds": 10, }, # Kanban multi-agent coordination — controls the dispatcher loop that diff --git a/tests/cron/test_sessiondb_init_hang.py b/tests/cron/test_sessiondb_init_hang.py index 2b18fdefb8ac..a98add9561b8 100644 --- a/tests/cron/test_sessiondb_init_hang.py +++ b/tests/cron/test_sessiondb_init_hang.py @@ -107,11 +107,63 @@ class TestSessionDbInitTimeout: mock_agent.run_conversation.return_value = {"final_response": "ok"} mock_agent_cls.return_value = mock_agent - success, output, final_response, error = run_job(job) + with caplog.at_level("WARNING"): + success, output, final_response, error = run_job(job) assert success is True kwargs = mock_agent_cls.call_args.kwargs assert kwargs["session_db"] is fake_db # default 10s was plenty for a MagicMock + # The malformed env var must produce a warning so the misconfiguration + # is observable — otherwise it silently falls back and operators can't + # diagnose why their custom timeout isn't taking effect. + assert any( + "HERMES_CRON_SESSION_DB_TIMEOUT" in rec.message + for rec in caplog.records + ), f"Expected warning about invalid timeout env var; got: {[r.message for r in caplog.records]}" + + def test_timeout_resolved_from_config_yaml(self, tmp_path, monkeypatch): + """cron.session_db_timeout_seconds in config.yaml is respected when + the env var is not set — the canonical config-first resolution path.""" + import yaml + + monkeypatch.delenv("HERMES_CRON_SESSION_DB_TIMEOUT", raising=False) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"cron": {"session_db_timeout_seconds": 0.2}}) + ) + never_set = threading.Event() + job = {"id": "config-timeout", "name": "test", "prompt": "hello"} + + try: + with patch("cron.scheduler._hermes_home", tmp_path), \ + patch("cron.scheduler._resolve_origin", return_value=None), \ + patch("hermes_cli.env_loader.load_hermes_dotenv"), \ + patch("hermes_cli.env_loader.reset_secret_source_cache"), \ + patch("hermes_state.SessionDB", side_effect=lambda: _hanging_session_db(never_set)), \ + patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + return_value={ + "api_key": "test-key", + "base_url": "https://example.invalid/v1", + "provider": "openrouter", + "api_mode": "chat_completions", + }, + ), \ + patch("run_agent.AIAgent") as mock_agent_cls: + mock_agent = MagicMock() + mock_agent.run_conversation.return_value = {"final_response": "ok"} + mock_agent_cls.return_value = mock_agent + + start = time.monotonic() + success, output, final_response, error = run_job(job) + elapsed = time.monotonic() - start + finally: + never_set.set() + + # Config value 0.2s bounds the hang, not the 10s default. + assert elapsed < 5.0 + assert success is True + assert mock_agent_cls.call_args.kwargs["session_db"] is None class TestDispatchGuardReleasedAfterHang: