mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-24 16:54:43 +00:00
f13f845116
Integration layer for the cjk_unicode61 tokenizer, rebuilt on the v23 schema (the contributed integration in PR #65544 predated it): - messages_fts_cjk: external-content FTS5 over a tool-row-excluding view (same v23 storage discipline as the trigram index it supersedes — zero inline text copies). Serves EVERY CJK query shape the legacy routing split between trigram (>=3 chars/token) and LIKE full scans (1-2 char tokens). Lone 1-char CJK runs and role_filter=['tool'] queries keep their legacy routes. - Dedicated marker pair (fts_cjk_rebuild_high_water/progress) gates the id-scoped triggers, so a cjk-only backfill never gates the complete messages_fts/trigram triggers. - Transitions ride (the existing throttled/resumable chunk engine): fresh DBs are born with the index; legacy v22 DBs land on v23+cjk in one run; already-optimized v23 DBs gaining the tokenizer get a marker-gated backfill; live writes are indexed immediately in every case. - Tokenizer-loss self-heal: a process that can't load the extension drops the cjk triggers (writes keep working), leaves a stale breadcrumb, and the index is rebuilt from scratch on the next optimize run — triggers are never reinstalled over a gap (external-content 'delete' on an unindexed rowid is the FTS5 corruption hazard the marker gating exists to prevent). - Capability classification: 'no such tokenizer: cjk_unicode61' joins the degraded-runtime error class everywhere (read probe, write probe, repair) so tokenizer absence is never misclassified as corruption. - Config: sessions.cjk_fts (default on, inert without the .so) and sessions.search_slow_ms in config.yaml, bridged to env by CLI + gateway (startup + per-turn reload). build.sh falls back to vendored SQLite headers so no libsqlite3-dev is needed. Slow-query log path attribution updated: fts_cjk / fts5 / trigram / like_scan. Tests: 14 lifecycle tests (fresh/legacy/stale/backfill paths, tokenizer-loss round-trip) + 5 config-bridge tests + slow-log suite.
60 lines
2.2 KiB
Python
60 lines
2.2 KiB
Python
"""Tests for the session-search slow-query log (salvaged from PR #65544)."""
|
|
|
|
import logging
|
|
|
|
import pytest
|
|
|
|
from hermes_state import SessionDB
|
|
|
|
|
|
@pytest.fixture()
|
|
def db(tmp_path):
|
|
d = SessionDB(db_path=tmp_path / "state.db")
|
|
d.create_session(session_id="s1", source="cli", model="m")
|
|
d.append_message("s1", role="user", content="hello graphiti 일본 MCP 정리")
|
|
yield d
|
|
d.close()
|
|
|
|
|
|
def test_slow_log_emitted_at_zero_threshold(db, monkeypatch, caplog):
|
|
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "0")
|
|
with caplog.at_level(logging.INFO, logger="hermes_state"):
|
|
rows = db.search_messages("graphiti", limit=5)
|
|
assert rows
|
|
slow = [r for r in caplog.records if "slow session search" in r.getMessage()]
|
|
assert slow, "threshold 0 must log every search"
|
|
msg = slow[0].getMessage()
|
|
assert "path=" in msg and "rows=1" in msg
|
|
|
|
|
|
def test_no_log_under_threshold(db, monkeypatch, caplog):
|
|
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "60000")
|
|
with caplog.at_level(logging.INFO, logger="hermes_state"):
|
|
db.search_messages("graphiti", limit=5)
|
|
assert not [r for r in caplog.records if "slow session search" in r.getMessage()]
|
|
|
|
|
|
def test_path_attribution(db):
|
|
# Without the cjk tokenizer loaded, routing matches the pre-cjk shape.
|
|
assert db._describe_search_path("graphiti OR neo4j") == "fts5"
|
|
assert db._describe_search_path("우선순위 캘린더") == "trigram"
|
|
assert db._describe_search_path("일본 MCP") == "like_scan"
|
|
|
|
|
|
def test_path_attribution_cjk_available(db):
|
|
# With the bigram index available, CJK queries (including 2-char terms)
|
|
# route to fts_cjk; lone 1-char CJK runs keep the LIKE route.
|
|
db._fts_cjk_available = True
|
|
try:
|
|
assert db._describe_search_path("일본 MCP") == "fts_cjk"
|
|
assert db._describe_search_path("우선순위 캘린더") == "fts_cjk"
|
|
assert db._describe_search_path("가 alone") == "like_scan"
|
|
assert db._describe_search_path("graphiti OR neo4j") == "fts5"
|
|
finally:
|
|
db._fts_cjk_available = False
|
|
|
|
|
|
def test_results_unchanged_by_wrapper(db, monkeypatch):
|
|
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "0")
|
|
rows = db.search_messages("graphiti", limit=5)
|
|
assert rows and rows[0]["session_id"] == "s1"
|