mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
fix(review): surface respawn-storm env vars in config.yaml + docs
Follow-up to PR #66479 salvage. The three new env vars (HERMES_LOCAL_STREAM_STALE_TIMEOUT, HERMES_GATEWAY_MAX_STARTS, HERMES_GATEWAY_START_WINDOW_S) were introduced as bare env-var reads with no config.yaml surface or documentation — violating the .env-is-for-secrets-only policy (behavioral settings must live in config.yaml, bridged to env internally). - config.yaml: add gateway.respawn_storm {max_starts, window_seconds} to DEFAULT_CONFIG, mirroring the existing restart_loop_guard pattern. - gateway.py: read config.yaml first, env vars override as escape-hatch. - environment-variables.md: document all three new env vars, noting the config.yaml alternative for the gateway ones. - chat_completion_helpers.py: cross-reference the env-var docs from the local stale timeout comment.
This commit is contained in:
kshitijk4poor
kshitij
parent
fdcf352797
commit
277eedefbe
5 changed files with 73 additions and 9 deletions
|
|
@ -3376,9 +3376,24 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
|||
# or deadlocked local endpoint stalled the session indefinitely). 900s
|
||||
# tolerates slow prefill while still bounding a hung endpoint. Applies
|
||||
# unless the user explicitly set HERMES_STREAM_STALE_TIMEOUT; override the
|
||||
# local ceiling with HERMES_LOCAL_STREAM_STALE_TIMEOUT.
|
||||
# local ceiling with HERMES_LOCAL_STREAM_STALE_TIMEOUT (documented in
|
||||
# website/docs/reference/environment-variables.md).
|
||||
if _stream_stale_timeout_base == 180.0 and agent.base_url and is_local_endpoint(agent.base_url):
|
||||
_stream_stale_timeout = env_float("HERMES_LOCAL_STREAM_STALE_TIMEOUT", 900.0)
|
||||
# Read config.yaml ``agent.local_stream_stale_timeout`` (default 900),
|
||||
# env var ``HERMES_LOCAL_STREAM_STALE_TIMEOUT`` overrides for escape-hatch.
|
||||
_local_default = 900.0
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
_cfg = load_config()
|
||||
_agent_cfg = _cfg.get("agent") if isinstance(_cfg, dict) else None
|
||||
if isinstance(_agent_cfg, dict):
|
||||
_v = _agent_cfg.get("local_stream_stale_timeout")
|
||||
if isinstance(_v, (int, float)):
|
||||
_local_default = float(_v)
|
||||
except Exception:
|
||||
pass
|
||||
_stream_stale_timeout = env_float("HERMES_LOCAL_STREAM_STALE_TIMEOUT", _local_default)
|
||||
logger.debug(
|
||||
"Local provider detected (%s) — stale stream timeout set to %.0fs",
|
||||
agent.base_url, _stream_stale_timeout,
|
||||
|
|
|
|||
|
|
@ -1165,6 +1165,13 @@ DEFAULT_CONFIG = {
|
|||
# default is 1800s) plus runtime slack. Set to 0 to disable the
|
||||
# gate and restore pre-fix behaviour (always inject).
|
||||
"gateway_auto_continue_freshness": 3600,
|
||||
# Stale-stream ceiling for local providers (Ollama, oMLX, llama-cpp) in
|
||||
# seconds. When the base stale timeout is at its default (180s) and a
|
||||
# local endpoint is detected, this finite ceiling replaces the former
|
||||
# infinite disable so a wedged local server eventually trips the
|
||||
# detector instead of hanging forever. The env var
|
||||
# ``HERMES_LOCAL_STREAM_STALE_TIMEOUT`` overrides for escape-hatch use.
|
||||
"local_stream_stale_timeout": 900,
|
||||
# How user-attached images are presented to the main model on each turn.
|
||||
# "auto" — attach natively when the active model reports
|
||||
# supports_vision=True AND the user hasn't explicitly
|
||||
|
|
@ -2988,6 +2995,19 @@ DEFAULT_CONFIG = {
|
|||
"window_seconds": 60,
|
||||
},
|
||||
|
||||
# Portable respawn-storm circuit breaker (complements
|
||||
# ``restart_loop_guard`` above). Counts gateway (re)starts in a sliding
|
||||
# window and, when too many land, sleeps an exponential backoff before
|
||||
# booting so a crash-looping supervisor (launchd KeepAlive, systemd
|
||||
# Restart=always) can't hammer the process into a respawn storm.
|
||||
# ``max_starts <= 0`` disables the breaker. The env vars
|
||||
# ``HERMES_GATEWAY_MAX_STARTS`` / ``HERMES_GATEWAY_START_WINDOW_S``
|
||||
# override these defaults for escape-hatch use.
|
||||
"respawn_storm": {
|
||||
"max_starts": 5,
|
||||
"window_seconds": 120,
|
||||
},
|
||||
|
||||
# Inject a human-readable timestamp prefix (e.g.
|
||||
# "[Tue 2026-04-28 13:40:53 CEST]") onto user messages IN THE MODEL'S
|
||||
# CONTEXT so the agent has temporal awareness of when each message was
|
||||
|
|
|
|||
|
|
@ -4868,21 +4868,46 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo
|
|||
|
||||
# Portable, app-level respawn-storm circuit breaker. launchd/systemd have
|
||||
# their own throttles, but this backstop works on every platform (and covers
|
||||
# supervisors that lack a respawn floor). Set HERMES_GATEWAY_MAX_STARTS<=0 to
|
||||
# disable. Kept best-effort: a bookkeeping failure must never block startup.
|
||||
# supervisors that lack a respawn floor). Configured via
|
||||
# ``gateway.respawn_storm`` in config.yaml (``max_starts`` / ``window_seconds``);
|
||||
# the env vars ``HERMES_GATEWAY_MAX_STARTS`` /
|
||||
# ``HERMES_GATEWAY_START_WINDOW_S`` override for escape-hatch use.
|
||||
# Set max_starts <= 0 to disable. Best-effort: a bookkeeping failure must
|
||||
# never block startup.
|
||||
try:
|
||||
import time as _time
|
||||
|
||||
from gateway.status import record_start_and_check_storm
|
||||
|
||||
# Defaults mirror config.yaml DEFAULT_CONFIG ``gateway.respawn_storm``.
|
||||
_max_starts = 5
|
||||
_win = 120.0
|
||||
try:
|
||||
_max_starts = int(os.getenv("HERMES_GATEWAY_MAX_STARTS", "5"))
|
||||
except ValueError:
|
||||
_max_starts = 5
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
_cfg = load_config()
|
||||
_gw = _cfg.get("gateway") if isinstance(_cfg, dict) else None
|
||||
_rs = _gw.get("respawn_storm") if isinstance(_gw, dict) else None
|
||||
if isinstance(_rs, dict):
|
||||
if isinstance(_rs.get("max_starts"), int):
|
||||
_max_starts = _rs["max_starts"]
|
||||
if isinstance(_rs.get("window_seconds"), (int, float)):
|
||||
_win = float(_rs["window_seconds"])
|
||||
except Exception:
|
||||
pass
|
||||
# Env vars override config for escape-hatch use.
|
||||
try:
|
||||
_win = float(os.getenv("HERMES_GATEWAY_START_WINDOW_S", "120"))
|
||||
_env_starts = os.getenv("HERMES_GATEWAY_MAX_STARTS")
|
||||
if _env_starts is not None:
|
||||
_max_starts = int(_env_starts)
|
||||
except ValueError:
|
||||
_win = 120.0
|
||||
pass
|
||||
try:
|
||||
_env_win = os.getenv("HERMES_GATEWAY_START_WINDOW_S")
|
||||
if _env_win is not None:
|
||||
_win = float(_env_win)
|
||||
except ValueError:
|
||||
pass
|
||||
_storm = (
|
||||
record_start_and_check_storm(max_starts=_max_starts, window_s=_win)
|
||||
if _max_starts > 0
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ LEGACY_AUTHOR_MAP = {
|
|||
"252620095+briandevans@users.noreply.github.com": "briandevans", # PR #64951 salvage (lmstudio: clamp max/ultra reasoning effort)
|
||||
"kar.iskakov@gmail.com": "karfly", # PR #64012 salvage (gateway: surface extended reasoning efforts)
|
||||
"kimyeon30@naver.com": "rlaehddus302", # PR #61985 salvage (gateway: secondary-adapter auth callback profile)
|
||||
"burke@autreymail.com": "bautrey", # PR #66479 salvage (gateway reliability hardening: Bedrock liveness, supervised watchers, launchd respawn throttle)
|
||||
"agungsubastian1963@gmail.com": "aguung", # PR #64461 salvage (gateway: multiplex secret_scope for authz/Slack/webhooks)
|
||||
"jtstothard@gmail.com": "jtstothard", # PR #63256 salvage (gateway: multiplex secondary adapter config validation)
|
||||
"fjlaowan@proton.me": "fjlaowan1983", # PR #11256 salvage (honcho: reject whitespace-only reasoning queries)
|
||||
|
|
|
|||
|
|
@ -734,9 +734,12 @@ Advanced per-platform knobs for throttling the outbound message batcher. Most us
|
|||
| `HERMES_API_CALL_STALE_TIMEOUT` | Non-streaming stale-call timeout in seconds (default: `90`). Auto-disabled for local providers when left unset, and may scale upward for very large contexts. Also configurable via `providers.<id>.stale_timeout_seconds` or `providers.<id>.models.<model>.stale_timeout_seconds` in `config.yaml`. |
|
||||
| `HERMES_STREAM_READ_TIMEOUT` | Streaming socket read timeout in seconds (default: `120`). Auto-increased to `HERMES_API_TIMEOUT` for local providers. Increase if local LLMs time out during long code generation. |
|
||||
| `HERMES_STREAM_STALE_TIMEOUT` | Stale stream detection timeout in seconds (default: `180`). Auto-disabled for local providers. Triggers connection kill if no chunks arrive within this window. |
|
||||
| `HERMES_LOCAL_STREAM_STALE_TIMEOUT` | Stale stream ceiling for local providers (Ollama, oMLX, llama-cpp) in seconds (default: `900`). When the base stale timeout is at its default and a local endpoint is detected, this finite ceiling replaces the former infinite disable so a wedged local server eventually trips the detector instead of hanging forever. Also configurable via `agent.local_stream_stale_timeout` in `config.yaml`. |
|
||||
| `HERMES_STREAM_RETRIES` | Number of mid-stream reconnect attempts on transient network errors (default: `3`). |
|
||||
| `HERMES_STREAM_STALE_GIVEUP` | Cross-turn circuit breaker: after this many consecutive stale kills (streaming or non-streaming) with no completed response, abort each call immediately with an actionable error instead of re-waiting out the stale timeout (default: `5`, `0` disables). Resets on any completed response, `/model` switch, fallback activation, or turn-start primary restore. |
|
||||
| `HERMES_AGENT_TIMEOUT` | Gateway inactivity timeout for a running agent in seconds (default: `1800`, 30 minutes). Resets on every tool call and streamed token. Set to `0` to disable. |
|
||||
| `HERMES_GATEWAY_MAX_STARTS` | Respawn-storm circuit breaker: maximum gateway (re)starts allowed within the window before an exponential backoff is slept to break the storm (default: `5`, `0` disables). Also configurable via `gateway.respawn_storm.max_starts` in `config.yaml`. |
|
||||
| `HERMES_GATEWAY_START_WINDOW_S` | Respawn-storm breaker window in seconds (default: `120`). Also configurable via `gateway.respawn_storm.window_seconds` in `config.yaml`. |
|
||||
| `HERMES_AGENT_TIMEOUT_WARNING` | Gateway: send a warning message after this many seconds of inactivity (default: 75% of `HERMES_AGENT_TIMEOUT`). |
|
||||
| `HERMES_AGENT_NOTIFY_INTERVAL` | Gateway: interval in seconds between progress notifications on long-running agent turns. |
|
||||
| `HERMES_CHECKPOINT_TIMEOUT` | Timeout for filesystem checkpoint creation in seconds (default: `30`). |
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue