diff --git a/hermes_state.py b/hermes_state.py index af4050a99e8..5ab828fd150 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -6777,6 +6777,102 @@ class SessionDB: run = 0 return run == 1 + @staticmethod + def _trigram_eligible_tokens(query: str) -> bool: + """True when every non-operator token is long enough for the trigram + tokenizer to match (>=3 chars). + + The trigram tokenizer indexes overlapping 3-character sequences, so a + token shorter than 3 chars produces no trigrams and can never match. + With FTS5's implicit-AND between tokens, a single short token makes the + whole MATCH return nothing, so the trigram path is only worth taking + when every searchable token qualifies. + """ + tokens = [ + t for t in query.strip('"').strip().split() + if t.upper() not in {"AND", "OR", "NOT"} + ] + return bool(tokens) and all(len(t) >= 3 for t in tokens) + + def _run_trigram_search( + self, + raw_query: str, + *, + table: str = "messages_fts_trigram", + order_by_sql: str, + include_inactive: bool, + source_filter: List[str] = None, + exclude_sources: List[str] = None, + role_filter: List[str] = None, + limit: int = 20, + offset: int = 0, + ) -> Optional[List[Dict[str, Any]]]: + """Run a search against a substring-capable FTS index. + + ``table`` is ``messages_fts_trigram`` (default) or + ``messages_fts_cjk``. The trigram tokenizer indexes overlapping + 3-byte sequences, so it matches substrings regardless of word + boundaries — both CJK phrases the unicode61 tokenizer splits into + single characters and Latin runs the unicode61 tokenizer fuses onto + adjacent CJK (e.g. ``修改youer服务端``). The cjk-bigram tokenizer + splits Latin runs off adjacent CJK, giving the same recovery as an + exact ranked token match. Each non-operator token is quoted to + neutralise FTS5 special characters while boolean operators + (AND/OR/NOT) are preserved. + + Returns the matching rows, or ``None`` when the query cannot be + executed (e.g. the tokenizer is unavailable at runtime) so the + caller can fall back to another strategy. + """ + tokens = raw_query.split() + parts = [] + for tok in tokens: + if tok.upper() in {"AND", "OR", "NOT"}: + parts.append(tok) + else: + parts.append('"' + tok.replace('"', '""') + '"') + trigram_query = " ".join(parts) + tri_where = [f"{table} MATCH ?"] + tri_params: list = [trigram_query] + if not include_inactive: + tri_where.append("(m.active = 1 OR m.compacted = 1)") + if source_filter is not None: + tri_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") + tri_params.extend(source_filter) + if exclude_sources is not None: + tri_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") + tri_params.extend(exclude_sources) + if role_filter: + tri_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") + tri_params.extend(role_filter) + tri_sql = f""" + SELECT + m.id, + m.session_id, + m.role, + snippet({table}, -1, '>>>', '<<<', '...', 40) AS snippet, + m.content, + m.timestamp, + m.tool_name, + s.source, + s.model, + s.started_at AS session_started + FROM {table} + JOIN messages m ON m.id = {table}.rowid + JOIN sessions s ON s.id = m.session_id + WHERE {' AND '.join(tri_where)} + {order_by_sql} + LIMIT ? OFFSET ? + """ + tri_params.extend([limit, offset]) + with self._lock: + try: + tri_cursor = self._conn.execute(tri_sql, tri_params) + except sqlite3.OperationalError: + # Query failed at runtime — let the caller fall back. + return None + return [dict(row) for row in tri_cursor.fetchall()] + def search_messages( self, query: str, @@ -7279,6 +7375,61 @@ class SessionDB: except sqlite3.OperationalError as exc: logger.debug("Unindexed-gap supplement skipped: %s", exc) + # Pure-Latin queries run against the unicode61 ``messages_fts`` table, + # whose tokenizer does not insert a boundary between Latin letters and + # adjacent CJK characters: "修改youer服务端" is indexed as one token, + # so MATCH "youer" finds nothing even though the substring is present + # (#54242). When the exact-token search returns nothing, retry on the + # substring-capable indexes. Preference order: + # 1. messages_fts_cjk (when built): its tokenizer splits Latin runs + # off adjacent CJK, so "youer" is an exact ranked token match. + # 2. messages_fts_trigram: substring matching, needs >=3-char + # tokens (shorter tokens produce no trigrams). + # Gated on a zero-result miss so successful Latin searches keep their + # unicode61 ranking — strictly additive, never reorders existing + # hits. Trade-off on the trigram leg: any zero-result Latin query + # gains substring semantics (e.g. "cat" can then match + # "concatenate"). Genuinely absent terms still return []. Skipped for + # role_filter=['tool'] queries — both fallback indexes exclude tool + # rows (v23), so a retry could never add hits. + if ( + not matches + and not is_cjk + and not (bool(role_filter) and "tool" in role_filter) + ): + _fb_query = query.strip('"').strip() + if self._fts_cjk_available: + cjk_fb = self._run_trigram_search( + _fb_query, + table="messages_fts_cjk", + order_by_sql=order_by_sql, + include_inactive=include_inactive, + source_filter=source_filter, + exclude_sources=exclude_sources, + role_filter=role_filter, + limit=limit, + offset=offset, + ) + if cjk_fb: + matches = cjk_fb + if ( + not matches + and self._trigram_available + and self._trigram_eligible_tokens(query) + ): + tri_matches = self._run_trigram_search( + _fb_query, + order_by_sql=order_by_sql, + include_inactive=include_inactive, + source_filter=source_filter, + exclude_sources=exclude_sources, + role_filter=role_filter, + limit=limit, + offset=offset, + ) + if tri_matches: + matches = tri_matches + # Add surrounding context (1 message before + after each match). # Done outside the lock so we don't hold it across N sequential queries. for match in matches: diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index 38222176b97..53f4b1dd8b3 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -2425,6 +2425,46 @@ class TestCJKSearchFallback: results = db.search_messages("Agent通信") assert len(results) == 1 + def test_pure_latin_word_embedded_in_cjk_is_found(self, db): + """Regression for #54242. + + A pure-Latin query (no CJK chars) routes to the unicode61 ``messages_fts`` + table, whose tokenizer fuses a Latin run onto the adjacent CJK characters + ("修改youer服务端" is indexed as a single token), so MATCH "youer" returns + nothing. The zero-result trigram fallback must recover the match. + """ + db.create_session(session_id="s1", source="cli") + db.append_message("s1", role="user", content="修改youer服务端的计划") + results = db.search_messages("youer") + assert len(results) == 1 + assert results[0]["session_id"] == "s1" + + def test_pure_latin_query_with_normal_match_is_unaffected(self, db): + """A normal space-delimited Latin query still resolves on the unicode61 + path; the zero-result trigram fallback only fires when it misses.""" + db.create_session(session_id="s1", source="cli") + db.append_message("s1", role="user", content="deploy the docker container") + results = db.search_messages("docker") + assert len(results) == 1 + assert results[0]["session_id"] == "s1" + + def test_pure_latin_query_absent_term_returns_empty(self, db): + """A Latin term that is genuinely absent must still return nothing, + even with the trigram fallback enabled.""" + db.create_session(session_id="s1", source="cli") + db.append_message("s1", role="user", content="修改youer服务端的计划") + assert db.search_messages("kubernetes") == [] + + def test_pure_latin_embedded_fallback_preserves_source_filter(self, db): + """The embedded-Latin trigram fallback must honour source_filter.""" + db.create_session(session_id="s1", source="cli") + db.create_session(session_id="s2", source="telegram") + db.append_message("s1", role="user", content="修改youer服务端cli") + db.append_message("s2", role="user", content="修改youer服务端telegram") + results = db.search_messages("youer", source_filter=["telegram"]) + assert len(results) == 1 + assert results[0]["source"] == "telegram" + def test_cjk_partial_fts5_results_supplemented_by_like(self, db): """When FTS5 returns *some* CJK results, LIKE must still find all matches.