diff --git a/gateway/run.py b/gateway/run.py index f6e86b29fd9a..15042bd4c394 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -2667,6 +2667,8 @@ def _normalize_empty_agent_response( ) return response if api_calls > 0: + if _is_gateway_hidden_reasoning_incomplete_turn(agent_result): + return "" if agent_result.get("partial"): err = agent_result.get("error", "processing incomplete") return f"⚠️ Processing stopped: {str(err)[:200]}. Try again." @@ -2694,6 +2696,20 @@ def _normalize_empty_agent_response( return response +def _is_gateway_hidden_reasoning_incomplete_turn(agent_result: dict) -> bool: + """Detect retry-exhausted turns with hidden reasoning but no visible answer.""" + if not isinstance(agent_result, dict): + return False + if agent_result.get("failed") or agent_result.get("interrupted"): + return False + if agent_result.get("final_response"): + return False + if not agent_result.get("partial"): + return False + error_text = str(agent_result.get("error", "") or "").lower() + return "remained incomplete after" in error_text + + def _should_clear_resume_pending_after_turn(agent_result: dict) -> bool: """Return True only when a gateway turn really completed successfully. @@ -12039,6 +12055,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # forgets what was just asked. Persist the user turn so the # conversation is preserved. (#7100) agent_failed_early = bool(agent_result.get("failed")) + hidden_reasoning_incomplete = _is_gateway_hidden_reasoning_incomplete_turn( + agent_result + ) _err_str_for_classify = str(agent_result.get("error", "")).lower() # Use specific multi-word phrases (not bare "exceed" or "token") # to avoid false positives on transient errors like "rate limit @@ -12067,6 +12086,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "message so conversation context is preserved on retry.", session_entry.session_id, ) + elif hidden_reasoning_incomplete: + logger.warning( + "Suppressing hidden-reasoning-only incomplete gateway turn " + "for session %s: %s", + session_entry.session_id, + agent_result.get("error", "processing incomplete"), + ) # When compression is exhausted, the session is permanently too # large to process. Auto-reset it so the next message starts @@ -12150,11 +12176,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # entries that were stripped before the agent saw them. if is_context_overflow_failure: pass # handled above — skip all transcript writes - elif agent_failed_early: + elif agent_failed_early or hidden_reasoning_incomplete: # Transient failure (429/timeout/5xx): persist only the user # message so the next message can load a transcript that # reflects what was said. Skip the assistant error text since - # it's a gateway-generated hint, not model output. (#7100) + # it's a gateway-generated hint, not model output. Hidden- + # reasoning-only incomplete turns follow the same persistence + # rule so peer-agent channels don't ingest them as completed + # assistant turns. (#7100, #51628) _user_entry = { "role": "user", "content": ( diff --git a/tests/gateway/test_incomplete_gateway_turns.py b/tests/gateway/test_incomplete_gateway_turns.py new file mode 100644 index 000000000000..1ac356711c36 --- /dev/null +++ b/tests/gateway/test_incomplete_gateway_turns.py @@ -0,0 +1,159 @@ +"""Regression tests for hidden-reasoning-only incomplete gateway turns.""" + +import asyncio +from datetime import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import gateway.run as gateway_run +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.platforms.base import BasePlatformAdapter, MessageEvent, ProcessingOutcome, SendResult +from gateway.session import SessionEntry, SessionSource, build_session_key + + +class CaptureSlackAdapter(BasePlatformAdapter): + def __init__(self): + super().__init__(PlatformConfig(enabled=True, token="fake-token"), Platform.SLACK) + self.sent = [] + self.processing_hooks = [] + + async def connect(self) -> bool: + return True + + async def disconnect(self) -> None: + return None + + async def send(self, chat_id, content, reply_to=None, metadata=None) -> SendResult: + self.sent.append( + { + "chat_id": chat_id, + "content": content, + "reply_to": reply_to, + "metadata": metadata, + } + ) + return SendResult(success=True, message_id="slack-1") + + async def send_typing(self, chat_id: str, metadata=None) -> None: + return None + + async def get_chat_info(self, chat_id: str): + return {"id": chat_id} + + async def on_processing_start(self, event: MessageEvent) -> None: + self.processing_hooks.append(("start", event.message_id)) + + async def on_processing_complete(self, event: MessageEvent, outcome: ProcessingOutcome) -> None: + self.processing_hooks.append(("complete", event.message_id, outcome)) + + +def _make_incomplete_result() -> dict: + return { + "final_response": None, + "messages": [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": ""}, + ], + "tools": [], + "history_offset": 0, + "api_calls": 3, + "partial": True, + "completed": False, + "interrupted": False, + "error": "Codex response remained incomplete after 3 continuation attempts", + "last_prompt_tokens": 0, + } + + +def _make_runner(adapter: CaptureSlackAdapter) -> gateway_run.GatewayRunner: + runner = object.__new__(gateway_run.GatewayRunner) + runner.config = GatewayConfig( + platforms={Platform.SLACK: PlatformConfig(enabled=True, token="fake-token")} + ) + runner.adapters = {Platform.SLACK: adapter} + runner._voice_mode = {} + runner.hooks = SimpleNamespace(emit=AsyncMock(), loaded_hooks=False) + runner.session_store = MagicMock() + runner.session_store.get_or_create_session.return_value = SessionEntry( + session_key="agent:main:slack:channel:C123:171717", + session_id="sess-1", + created_at=datetime.now(), + updated_at=datetime.now(), + platform=Platform.SLACK, + chat_type="channel", + ) + runner.session_store.load_transcript.return_value = [] + runner.session_store.has_any_sessions.return_value = True + runner.session_store.rewrite_transcript = MagicMock() + runner.session_store.append_to_transcript = MagicMock() + runner.session_store.update_session = MagicMock() + runner._running_agents = {} + runner._pending_messages = {} + runner._pending_approvals = {} + runner._session_db = None + runner._is_user_authorized = lambda _source: True + runner._set_session_env = lambda _context: None + runner._run_agent = AsyncMock(return_value=_make_incomplete_result()) + return runner + + +def _make_event() -> MessageEvent: + return MessageEvent( + text="hello", + source=SessionSource( + platform=Platform.SLACK, + chat_id="C123", + chat_type="channel", + thread_id="171717", + user_id="U123", + ), + message_id="m-1", + ) + + +def test_incomplete_codex_warning_is_not_surfaced_as_chat_text(): + agent_result = _make_incomplete_result() + + response = gateway_run._normalize_empty_agent_response( + agent_result, + agent_result.get("final_response") or "", + history_len=4, + ) + + assert response == "" + + +@pytest.mark.asyncio +async def test_incomplete_codex_turn_stays_out_of_slack_transcript(monkeypatch, tmp_path): + adapter = CaptureSlackAdapter() + runner = _make_runner(adapter) + + monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path) + monkeypatch.setattr(gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"}) + monkeypatch.setattr( + "agent.model_metadata.get_model_context_length", + lambda *_args, **_kwargs: 100, + ) + monkeypatch.setenv("SLACK_HOME_CHANNEL", "C123") + + adapter.set_message_handler(runner._handle_message) + adapter._keep_typing = lambda *_args, **_kwargs: asyncio.Event().wait() + + event = _make_event() + await adapter._process_message_background(event, build_session_key(event.source)) + + assert adapter.sent == [] + assert runner.session_store.update_session.called + + transcript_roles = [ + call.args[1]["role"] + for call in runner.session_store.append_to_transcript.call_args_list + ] + assert transcript_roles == ["session_meta", "user"] + assert runner.session_store.append_to_transcript.call_args_list[1].args[1]["content"] == "hello" + assert adapter.processing_hooks == [ + ("start", "m-1"), + ("complete", "m-1", ProcessingOutcome.SUCCESS), + ] diff --git a/website/docs/user-guide/messaging/slack.md b/website/docs/user-guide/messaging/slack.md index 42acf9f3edb7..299b5fa0ad89 100644 --- a/website/docs/user-guide/messaging/slack.md +++ b/website/docs/user-guide/messaging/slack.md @@ -217,6 +217,12 @@ hermes gateway install # Install as a user service sudo hermes gateway install --system # Linux only: boot-time system service ``` +:::tip Codex reasoning-effort safety +For Codex-backed Slack peer-agent channels, prefer `agent.reasoning_effort: high` or lower. `xhigh` +can spend the entire turn in hidden reasoning and never produce visible assistant text; Hermes now +suppresses those incomplete-turn warnings from the thread and keeps the diagnostics in gateway logs. +::: + --- ## Step 9: Invite the Bot to Channels