From 277eedefbea5707e304f62dd4062d269c9c676cf Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Sat, 18 Jul 2026 19:26:16 +0530 Subject: [PATCH] fix(review): surface respawn-storm env vars in config.yaml + docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to PR #66479 salvage. The three new env vars (HERMES_LOCAL_STREAM_STALE_TIMEOUT, HERMES_GATEWAY_MAX_STARTS, HERMES_GATEWAY_START_WINDOW_S) were introduced as bare env-var reads with no config.yaml surface or documentation — violating the .env-is-for-secrets-only policy (behavioral settings must live in config.yaml, bridged to env internally). - config.yaml: add gateway.respawn_storm {max_starts, window_seconds} to DEFAULT_CONFIG, mirroring the existing restart_loop_guard pattern. - gateway.py: read config.yaml first, env vars override as escape-hatch. - environment-variables.md: document all three new env vars, noting the config.yaml alternative for the gateway ones. - chat_completion_helpers.py: cross-reference the env-var docs from the local stale timeout comment. --- agent/chat_completion_helpers.py | 19 ++++++++- hermes_cli/config.py | 20 ++++++++++ hermes_cli/gateway.py | 39 +++++++++++++++---- scripts/release.py | 1 + .../docs/reference/environment-variables.md | 3 ++ 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index f1816568db50..36522e494d86 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -3376,9 +3376,24 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # or deadlocked local endpoint stalled the session indefinitely). 900s # tolerates slow prefill while still bounding a hung endpoint. Applies # unless the user explicitly set HERMES_STREAM_STALE_TIMEOUT; override the - # local ceiling with HERMES_LOCAL_STREAM_STALE_TIMEOUT. + # local ceiling with HERMES_LOCAL_STREAM_STALE_TIMEOUT (documented in + # website/docs/reference/environment-variables.md). if _stream_stale_timeout_base == 180.0 and agent.base_url and is_local_endpoint(agent.base_url): - _stream_stale_timeout = env_float("HERMES_LOCAL_STREAM_STALE_TIMEOUT", 900.0) + # Read config.yaml ``agent.local_stream_stale_timeout`` (default 900), + # env var ``HERMES_LOCAL_STREAM_STALE_TIMEOUT`` overrides for escape-hatch. + _local_default = 900.0 + try: + from hermes_cli.config import load_config + + _cfg = load_config() + _agent_cfg = _cfg.get("agent") if isinstance(_cfg, dict) else None + if isinstance(_agent_cfg, dict): + _v = _agent_cfg.get("local_stream_stale_timeout") + if isinstance(_v, (int, float)): + _local_default = float(_v) + except Exception: + pass + _stream_stale_timeout = env_float("HERMES_LOCAL_STREAM_STALE_TIMEOUT", _local_default) logger.debug( "Local provider detected (%s) — stale stream timeout set to %.0fs", agent.base_url, _stream_stale_timeout, diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 1b0ea059317b..0bbd1f703c47 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1165,6 +1165,13 @@ DEFAULT_CONFIG = { # default is 1800s) plus runtime slack. Set to 0 to disable the # gate and restore pre-fix behaviour (always inject). "gateway_auto_continue_freshness": 3600, + # Stale-stream ceiling for local providers (Ollama, oMLX, llama-cpp) in + # seconds. When the base stale timeout is at its default (180s) and a + # local endpoint is detected, this finite ceiling replaces the former + # infinite disable so a wedged local server eventually trips the + # detector instead of hanging forever. The env var + # ``HERMES_LOCAL_STREAM_STALE_TIMEOUT`` overrides for escape-hatch use. + "local_stream_stale_timeout": 900, # How user-attached images are presented to the main model on each turn. # "auto" — attach natively when the active model reports # supports_vision=True AND the user hasn't explicitly @@ -2988,6 +2995,19 @@ DEFAULT_CONFIG = { "window_seconds": 60, }, + # Portable respawn-storm circuit breaker (complements + # ``restart_loop_guard`` above). Counts gateway (re)starts in a sliding + # window and, when too many land, sleeps an exponential backoff before + # booting so a crash-looping supervisor (launchd KeepAlive, systemd + # Restart=always) can't hammer the process into a respawn storm. + # ``max_starts <= 0`` disables the breaker. The env vars + # ``HERMES_GATEWAY_MAX_STARTS`` / ``HERMES_GATEWAY_START_WINDOW_S`` + # override these defaults for escape-hatch use. + "respawn_storm": { + "max_starts": 5, + "window_seconds": 120, + }, + # Inject a human-readable timestamp prefix (e.g. # "[Tue 2026-04-28 13:40:53 CEST]") onto user messages IN THE MODEL'S # CONTEXT so the agent has temporal awareness of when each message was diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index 0984f88a8c86..9868b976c97d 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -4868,21 +4868,46 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo # Portable, app-level respawn-storm circuit breaker. launchd/systemd have # their own throttles, but this backstop works on every platform (and covers - # supervisors that lack a respawn floor). Set HERMES_GATEWAY_MAX_STARTS<=0 to - # disable. Kept best-effort: a bookkeeping failure must never block startup. + # supervisors that lack a respawn floor). Configured via + # ``gateway.respawn_storm`` in config.yaml (``max_starts`` / ``window_seconds``); + # the env vars ``HERMES_GATEWAY_MAX_STARTS`` / + # ``HERMES_GATEWAY_START_WINDOW_S`` override for escape-hatch use. + # Set max_starts <= 0 to disable. Best-effort: a bookkeeping failure must + # never block startup. try: import time as _time from gateway.status import record_start_and_check_storm + # Defaults mirror config.yaml DEFAULT_CONFIG ``gateway.respawn_storm``. + _max_starts = 5 + _win = 120.0 try: - _max_starts = int(os.getenv("HERMES_GATEWAY_MAX_STARTS", "5")) - except ValueError: - _max_starts = 5 + from hermes_cli.config import load_config + + _cfg = load_config() + _gw = _cfg.get("gateway") if isinstance(_cfg, dict) else None + _rs = _gw.get("respawn_storm") if isinstance(_gw, dict) else None + if isinstance(_rs, dict): + if isinstance(_rs.get("max_starts"), int): + _max_starts = _rs["max_starts"] + if isinstance(_rs.get("window_seconds"), (int, float)): + _win = float(_rs["window_seconds"]) + except Exception: + pass + # Env vars override config for escape-hatch use. try: - _win = float(os.getenv("HERMES_GATEWAY_START_WINDOW_S", "120")) + _env_starts = os.getenv("HERMES_GATEWAY_MAX_STARTS") + if _env_starts is not None: + _max_starts = int(_env_starts) except ValueError: - _win = 120.0 + pass + try: + _env_win = os.getenv("HERMES_GATEWAY_START_WINDOW_S") + if _env_win is not None: + _win = float(_env_win) + except ValueError: + pass _storm = ( record_start_and_check_storm(max_starts=_max_starts, window_s=_win) if _max_starts > 0 diff --git a/scripts/release.py b/scripts/release.py index 5c72d2bd5ad5..cc547ff6a5f1 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -57,6 +57,7 @@ LEGACY_AUTHOR_MAP = { "252620095+briandevans@users.noreply.github.com": "briandevans", # PR #64951 salvage (lmstudio: clamp max/ultra reasoning effort) "kar.iskakov@gmail.com": "karfly", # PR #64012 salvage (gateway: surface extended reasoning efforts) "kimyeon30@naver.com": "rlaehddus302", # PR #61985 salvage (gateway: secondary-adapter auth callback profile) + "burke@autreymail.com": "bautrey", # PR #66479 salvage (gateway reliability hardening: Bedrock liveness, supervised watchers, launchd respawn throttle) "agungsubastian1963@gmail.com": "aguung", # PR #64461 salvage (gateway: multiplex secret_scope for authz/Slack/webhooks) "jtstothard@gmail.com": "jtstothard", # PR #63256 salvage (gateway: multiplex secondary adapter config validation) "fjlaowan@proton.me": "fjlaowan1983", # PR #11256 salvage (honcho: reject whitespace-only reasoning queries) diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 0e2811bcae29..aab4cb6a4906 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -734,9 +734,12 @@ Advanced per-platform knobs for throttling the outbound message batcher. Most us | `HERMES_API_CALL_STALE_TIMEOUT` | Non-streaming stale-call timeout in seconds (default: `90`). Auto-disabled for local providers when left unset, and may scale upward for very large contexts. Also configurable via `providers..stale_timeout_seconds` or `providers..models..stale_timeout_seconds` in `config.yaml`. | | `HERMES_STREAM_READ_TIMEOUT` | Streaming socket read timeout in seconds (default: `120`). Auto-increased to `HERMES_API_TIMEOUT` for local providers. Increase if local LLMs time out during long code generation. | | `HERMES_STREAM_STALE_TIMEOUT` | Stale stream detection timeout in seconds (default: `180`). Auto-disabled for local providers. Triggers connection kill if no chunks arrive within this window. | +| `HERMES_LOCAL_STREAM_STALE_TIMEOUT` | Stale stream ceiling for local providers (Ollama, oMLX, llama-cpp) in seconds (default: `900`). When the base stale timeout is at its default and a local endpoint is detected, this finite ceiling replaces the former infinite disable so a wedged local server eventually trips the detector instead of hanging forever. Also configurable via `agent.local_stream_stale_timeout` in `config.yaml`. | | `HERMES_STREAM_RETRIES` | Number of mid-stream reconnect attempts on transient network errors (default: `3`). | | `HERMES_STREAM_STALE_GIVEUP` | Cross-turn circuit breaker: after this many consecutive stale kills (streaming or non-streaming) with no completed response, abort each call immediately with an actionable error instead of re-waiting out the stale timeout (default: `5`, `0` disables). Resets on any completed response, `/model` switch, fallback activation, or turn-start primary restore. | | `HERMES_AGENT_TIMEOUT` | Gateway inactivity timeout for a running agent in seconds (default: `1800`, 30 minutes). Resets on every tool call and streamed token. Set to `0` to disable. | +| `HERMES_GATEWAY_MAX_STARTS` | Respawn-storm circuit breaker: maximum gateway (re)starts allowed within the window before an exponential backoff is slept to break the storm (default: `5`, `0` disables). Also configurable via `gateway.respawn_storm.max_starts` in `config.yaml`. | +| `HERMES_GATEWAY_START_WINDOW_S` | Respawn-storm breaker window in seconds (default: `120`). Also configurable via `gateway.respawn_storm.window_seconds` in `config.yaml`. | | `HERMES_AGENT_TIMEOUT_WARNING` | Gateway: send a warning message after this many seconds of inactivity (default: 75% of `HERMES_AGENT_TIMEOUT`). | | `HERMES_AGENT_NOTIFY_INTERVAL` | Gateway: interval in seconds between progress notifications on long-running agent turns. | | `HERMES_CHECKPOINT_TIMEOUT` | Timeout for filesystem checkpoint creation in seconds (default: `30`). |