"""Durable delivery-obligation ledger for gateway final responses. A final agent response that was generated but not yet confirmed-delivered to the messaging platform is the one artifact the gateway can lose without a trace: the turn already burned its tokens, the text exists only in a Python local, and a crash / planned restart between finalize and platform ACK drops it silently (#58818, #41696, #63695). This module records a small durable row per outbound final response in the shared ``state.db`` (same file and conventions as ``tools.async_delegation`` — WAL, owner pid + process-start-time liveness, bounded retention). The gateway writes three checkpoints around the send: record_obligation() state='pending' before any send attempt mark_attempting() state='attempting' immediately before the await mark_delivered() / state='delivered' only on SendResult.success mark_failed() state='failed' on a definitive rejection On startup, ``sweep_recoverable()`` claims rows whose owning process is dead and hands them to the gateway for redelivery. Crash semantics are explicit about ambiguity (the contract review of the earlier delivery-outbox attempt, #61790, closed it for silently resending ambiguous sends): - ``pending`` — the send never started: redeliver plainly, no dup risk. - ``attempting`` — crashed mid-await: the platform MAY already have the message. Redelivered WITH a visible recovered-reply marker so the contract is honest at-least-once, never a silent duplicate. - ``failed`` — definitively rejected once; the restart is a natural retry boundary. Also carries the marker. - ``delivered`` — nothing to do; retention prunes. Poison rows cannot spin: attempts are capped, stale rows expire, and both transition to ``abandoned`` (kept briefly for inspection, then pruned). Everything here is best-effort by design: ledger failures must never block or delay an actual send. Callers wrap every call in try/except. """ from __future__ import annotations import hashlib import json import logging import os import sqlite3 import threading import time from typing import Any, Dict, List, Optional from hermes_constants import get_hermes_home logger = logging.getLogger(__name__) _DB_LOCK = threading.Lock() # Redelivery policy knobs (module constants; deliberately not config — the # ledger itself is gated by ``gateway.delivery_ledger`` and these bounds # only matter in the rare recovery path). MAX_ATTEMPTS = 3 STALE_AFTER_SECONDS = 24 * 60 * 60 _RETENTION_SECONDS = 7 * 24 * 60 * 60 _MAX_ROWS = 500 # Visible prefix for redeliveries that might duplicate an already-received # message (crash mid-send / post-rejection retry). Honest at-least-once. RECOVERED_MARKER = ( "♻️ Recovered reply — the gateway restarted during delivery, " "so this may be a duplicate:\n\n" ) def _db_path(): return get_hermes_home() / "state.db" def _connect() -> sqlite3.Connection: from hermes_state import apply_wal_with_fallback path = _db_path() path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path, timeout=10) apply_wal_with_fallback(conn, db_label="state.db (delivery_ledger)") conn.execute( """CREATE TABLE IF NOT EXISTS delivery_obligations ( obligation_id TEXT PRIMARY KEY, session_key TEXT NOT NULL, platform TEXT NOT NULL, chat_id TEXT NOT NULL, thread_id TEXT, content TEXT NOT NULL, state TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL, owner_pid INTEGER, owner_started_at INTEGER, last_error TEXT )""" ) return conn def _owner_stamp() -> tuple[int, Optional[int]]: pid = os.getpid() try: from gateway.status import get_process_start_time return pid, get_process_start_time(pid) except Exception: return pid, None def _owner_alive(pid: Any, started_at: Any) -> bool: """True when the recorded owning process still exists (pid + start time).""" if not pid: return False try: pid = int(pid) except (TypeError, ValueError): return False try: from gateway.status import get_process_start_time current_start = get_process_start_time(pid) except Exception: current_start = None if current_start is None: # No such process (or unreadable) — treat unreadable-but-extant # processes as alive only if the pid exists. try: os.kill(pid, 0) # windows-footgun: ok — EPERM counts as alive below except ProcessLookupError: return False except PermissionError: return True except OSError: return False return True if started_at is None: return True try: return int(current_start) == int(started_at) except (TypeError, ValueError): return True def compute_obligation_id(session_key: str, message_ref: str, content: str) -> str: """Stable id: same turn + same content re-records idempotently, while distinct threads/topics on the same chat can never collide (the session_key carries platform, chat and thread; ``message_ref`` is the triggering inbound message id, distinguishing turns in one session).""" payload = f"{session_key}|{message_ref}|{content}" return hashlib.sha256(payload.encode("utf-8", "replace")).hexdigest()[:24] def record_obligation( *, obligation_id: str, session_key: str, platform: str, chat_id: str, thread_id: Optional[str], content: str, ) -> None: """Record a final response as owed to the platform (state='pending').""" now = time.time() pid, started = _owner_stamp() with _DB_LOCK, _connect() as conn: conn.execute( """INSERT OR REPLACE INTO delivery_obligations (obligation_id, session_key, platform, chat_id, thread_id, content, state, attempts, created_at, updated_at, owner_pid, owner_started_at) VALUES (?, ?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?, ?)""", (obligation_id, session_key, platform, str(chat_id), str(thread_id) if thread_id else None, content, now, now, pid, started), ) _prune() def mark_attempting(obligation_id: str) -> None: _update_state(obligation_id, "attempting") def mark_delivered(obligation_id: str) -> None: _update_state(obligation_id, "delivered") def mark_failed(obligation_id: str, error: str = "") -> None: _update_state(obligation_id, "failed", error=error) def _update_state(obligation_id: str, state: str, error: str = "") -> None: with _DB_LOCK, _connect() as conn: conn.execute( """UPDATE delivery_obligations SET state=?, updated_at=?, last_error=? WHERE obligation_id=?""", (state, time.time(), error[:500] if error else None, obligation_id), ) def sweep_recoverable( now: Optional[float] = None, *, deliverable_platforms: Optional[set] = None, ) -> List[Dict[str, Any]]: """Claim undelivered rows owned by dead processes; return them for redelivery. Claiming atomically re-stamps the owner to THIS process and increments ``attempts``, so a second gateway racing the same sweep cannot double-claim (the UPDATE is guarded on the previous owner stamp). Rows over the attempts cap or older than the stale cutoff transition to 'abandoned' instead of being returned. ``deliverable_platforms`` (platform value strings) restricts claiming to platforms the caller can actually send on this boot. ``attempts`` is the redelivery budget, so it must only be spent on a real send: a platform that failed to connect would otherwise burn one attempt per boot and hit the cap having never been sent once. Rows for absent platforms are left untouched for a later boot; the stale cutoff still bounds them. """ now = now if now is not None else time.time() pid, started = _owner_stamp() claimed: List[Dict[str, Any]] = [] with _DB_LOCK, _connect() as conn: rows = conn.execute( """SELECT obligation_id, session_key, platform, chat_id, thread_id, content, state, attempts, created_at, owner_pid, owner_started_at FROM delivery_obligations WHERE state IN ('pending', 'attempting', 'failed')""" ).fetchall() for (oid, session_key, platform, chat_id, thread_id, content, state, attempts, created_at, owner_pid, owner_started_at) in rows: if _owner_alive(owner_pid, owner_started_at): continue # a live gateway still owns this row if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: conn.execute( """UPDATE delivery_obligations SET state='abandoned', updated_at=? WHERE obligation_id=?""", (now, oid), ) continue if ( deliverable_platforms is not None and platform not in deliverable_platforms ): # No adapter for this platform this boot — the caller cannot # send, so claiming would spend an attempt on a no-op. continue cursor = conn.execute( """UPDATE delivery_obligations SET owner_pid=?, owner_started_at=?, attempts=attempts+1, updated_at=? WHERE obligation_id=? AND (owner_pid IS ? OR owner_pid=?)""", (pid, started, now, oid, owner_pid, owner_pid), ) if cursor.rowcount: claimed.append({ "obligation_id": oid, "session_key": session_key, "platform": platform, "chat_id": chat_id, "thread_id": thread_id, "content": content, # pending = send never started, redeliver plainly; # attempting/failed = ambiguous or rejected, carry marker. "needs_marker": state != "pending", "attempts": attempts + 1, }) return claimed def _prune(now: Optional[float] = None) -> None: now = now if now is not None else time.time() cutoff = now - _RETENTION_SECONDS try: with _connect() as conn: conn.execute( """DELETE FROM delivery_obligations WHERE state IN ('delivered', 'abandoned') AND updated_at < ?""", (cutoff,), ) total = conn.execute( "SELECT COUNT(*) FROM delivery_obligations" ).fetchone()[0] excess = max(0, total - _MAX_ROWS) if excess: conn.execute( """DELETE FROM delivery_obligations WHERE obligation_id IN ( SELECT obligation_id FROM delivery_obligations ORDER BY CASE state WHEN 'delivered' THEN 0 WHEN 'abandoned' THEN 1 ELSE 2 END, updated_at ASC LIMIT ?)""", (excess,), ) except Exception: logger.debug("delivery ledger prune failed", exc_info=True) def ledger_enabled(config: Optional[Dict[str, Any]] = None) -> bool: """Read the ``gateway.delivery_ledger`` config gate (default on).""" try: if config is None: from hermes_cli.config import load_config config = load_config() gw = config.get("gateway") or {} value = gw.get("delivery_ledger", True) if isinstance(value, str): return value.strip().lower() not in {"false", "0", "no", "off"} return bool(value) except Exception: return True def debug_rows(limit: int = 20) -> str: """Human-readable dump for ad-hoc inspection (sqlite3-free path).""" with _DB_LOCK, _connect() as conn: rows = conn.execute( """SELECT obligation_id, session_key, state, attempts, created_at, updated_at, last_error FROM delivery_obligations ORDER BY updated_at DESC LIMIT ?""", (limit,), ).fetchall() return json.dumps( [ { "id": r[0], "session": r[1], "state": r[2], "attempts": r[3], "created_at": r[4], "updated_at": r[5], "last_error": r[6], } for r in rows ], indent=2, )