mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
6b81590c55
Systematic prune per AGENTS.md test policy, one pass over every major test tree (gateway, hermes_cli, tools, agent, run_agent, plugins, cli, cron, tui_gateway, honcho/openviking, root-level): - DELETE: source-reading tests (read_text/getsource on prod files), change-detector tests (exact catalog counts, model-name snapshots, config version literals), mock-echo tests (assert a mock returns what it was told), assertion-free/trivial tests, near-duplicate parametrizations (boundaries + one representative kept), async/sync twin duplicates, cosmetic within-file variations. - KEEP (mandatory): security/redaction/approval guards, message-role alternation invariants, prompt-caching/deterministic-call-id invariants, issue-number regression tests (deduped), E2E tests. - 6 test files deleted outright (script-style/no-assert or fully redundant); conftest.py, fakes/, fixtures/ untouched. - tests/acp/conftest.py added: autouse fixture stubs the live models.dev/GitHub/Copilot/Anthropic inventory fetches that ACP server tests performed on every session create — test_server.py 147s → 3.4s, and the tests are now genuinely hermetic. - Sleep-based slowness shrunk where safe (codex_ttfb_watchdog, compression_concurrent_fork, etc.); no wall-clock assertion tightened. Verification: full hermetic suite via scripts/run_tests.sh — 2439 files, 31,130 tests passed, 0 failed, 0 flaky retries, 315s wall (baseline: 583s wall, 13,564s subprocess CPU).
517 lines
20 KiB
Python
517 lines
20 KiB
Python
"""Tests for agent/insights.py — InsightsEngine analytics and reporting."""
|
|
|
|
import time
|
|
import pytest
|
|
|
|
from hermes_state import SessionDB
|
|
from agent.insights import (
|
|
InsightsEngine,
|
|
_estimate_cost,
|
|
_bar_chart,
|
|
)
|
|
from agent.usage_pricing import (
|
|
format_duration_compact as _format_duration,
|
|
has_known_pricing as _has_known_pricing,
|
|
)
|
|
|
|
|
|
@pytest.fixture()
|
|
def db(tmp_path):
|
|
"""Create a SessionDB with a temp database file."""
|
|
db_path = tmp_path / "test_insights.db"
|
|
session_db = SessionDB(db_path=db_path)
|
|
yield session_db
|
|
session_db.close()
|
|
|
|
|
|
@pytest.fixture()
|
|
def populated_db(db):
|
|
"""Create a DB with realistic session data for insights testing."""
|
|
now = time.time()
|
|
day = 86400
|
|
|
|
# Session 1: CLI, claude-sonnet, ended, 2 days ago
|
|
db.create_session(
|
|
session_id="s1", source="cli",
|
|
model="anthropic/claude-sonnet-4-20250514", user_id="user1",
|
|
)
|
|
# Backdate the started_at
|
|
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's1'", (now - 2 * day,))
|
|
db.end_session("s1", end_reason="user_exit")
|
|
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's1'", (now - 2 * day + 3600,))
|
|
db.update_token_counts("s1", input_tokens=50000, output_tokens=15000)
|
|
db.append_message("s1", role="user", content="Hello, help me fix a bug")
|
|
db.append_message("s1", role="assistant", content="Sure, let me look into that.")
|
|
db.append_message("s1", role="assistant", content="Let me search the files.",
|
|
tool_calls=[{"function": {"name": "search_files"}}])
|
|
db.append_message("s1", role="tool", content="Found 3 matches", tool_name="search_files")
|
|
db.append_message("s1", role="assistant", content="Let me read the file.",
|
|
tool_calls=[{"function": {"name": "read_file"}}])
|
|
db.append_message("s1", role="tool", content="file contents...", tool_name="read_file")
|
|
db.append_message("s1", role="assistant", content="I found the bug. Let me fix it.",
|
|
tool_calls=[{"function": {"name": "patch"}}])
|
|
db.append_message("s1", role="tool", content="patched successfully", tool_name="patch")
|
|
db.append_message(
|
|
"s1",
|
|
role="assistant",
|
|
content="Let me load the PR workflow skill.",
|
|
tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}}],
|
|
)
|
|
db.append_message("s1", role="user", content="Thanks!")
|
|
db.append_message("s1", role="assistant", content="You're welcome!")
|
|
|
|
# Session 2: Telegram, gpt-4o, ended, 5 days ago
|
|
db.create_session(
|
|
session_id="s2", source="telegram",
|
|
model="gpt-4o", user_id="user1",
|
|
)
|
|
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's2'", (now - 5 * day,))
|
|
db.end_session("s2", end_reason="timeout")
|
|
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's2'", (now - 5 * day + 1800,))
|
|
db.update_token_counts("s2", input_tokens=20000, output_tokens=8000)
|
|
db.append_message("s2", role="user", content="Search the web for something")
|
|
db.append_message("s2", role="assistant", content="Searching...",
|
|
tool_calls=[{"function": {"name": "web_search"}}])
|
|
db.append_message("s2", role="tool", content="results...", tool_name="web_search")
|
|
db.append_message("s2", role="assistant", content="Here's what I found")
|
|
|
|
# Session 3: CLI, deepseek-chat, ended, 10 days ago
|
|
db.create_session(
|
|
session_id="s3", source="cli",
|
|
model="deepseek-chat", user_id="user1",
|
|
)
|
|
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's3'", (now - 10 * day,))
|
|
db.end_session("s3", end_reason="user_exit")
|
|
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's3'", (now - 10 * day + 7200,))
|
|
db.update_token_counts("s3", input_tokens=100000, output_tokens=40000)
|
|
db.append_message("s3", role="user", content="Run this terminal command")
|
|
db.append_message("s3", role="assistant", content="Running...",
|
|
tool_calls=[{"function": {"name": "terminal"}}])
|
|
db.append_message("s3", role="tool", content="output...", tool_name="terminal")
|
|
db.append_message("s3", role="assistant", content="Let me run another",
|
|
tool_calls=[{"function": {"name": "terminal"}}])
|
|
db.append_message("s3", role="tool", content="more output...", tool_name="terminal")
|
|
db.append_message("s3", role="assistant", content="And search files",
|
|
tool_calls=[{"function": {"name": "search_files"}}])
|
|
db.append_message("s3", role="tool", content="found stuff", tool_name="search_files")
|
|
db.append_message(
|
|
"s3",
|
|
role="assistant",
|
|
content="Load the debugging skill.",
|
|
tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"systematic-debugging"}'}}],
|
|
)
|
|
|
|
# Session 4: Discord, same model as s1, ended, 1 day ago
|
|
db.create_session(
|
|
session_id="s4", source="discord",
|
|
model="anthropic/claude-sonnet-4-20250514", user_id="user2",
|
|
)
|
|
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's4'", (now - 1 * day,))
|
|
db.end_session("s4", end_reason="user_exit")
|
|
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's4'", (now - 1 * day + 900,))
|
|
db.update_token_counts("s4", input_tokens=10000, output_tokens=5000)
|
|
db.append_message("s4", role="user", content="Quick question")
|
|
db.append_message("s4", role="assistant", content="Sure, go ahead")
|
|
db.append_message(
|
|
"s4",
|
|
role="assistant",
|
|
content="Load and update GitHub skills.",
|
|
tool_calls=[
|
|
{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}},
|
|
{"function": {"name": "skill_manage", "arguments": '{"name":"github-code-review"}'}},
|
|
],
|
|
)
|
|
|
|
# Session 5: Old session, 45 days ago (should be excluded from 30-day window)
|
|
db.create_session(
|
|
session_id="s_old", source="cli",
|
|
model="gpt-4o-mini", user_id="user1",
|
|
)
|
|
db._conn.execute("UPDATE sessions SET started_at = ? WHERE id = 's_old'", (now - 45 * day,))
|
|
db.end_session("s_old", end_reason="user_exit")
|
|
db._conn.execute("UPDATE sessions SET ended_at = ? WHERE id = 's_old'", (now - 45 * day + 600,))
|
|
db.update_token_counts("s_old", input_tokens=5000, output_tokens=2000)
|
|
db.append_message("s_old", role="user", content="old message")
|
|
db.append_message("s_old", role="assistant", content="old reply")
|
|
|
|
db._conn.commit()
|
|
return db
|
|
|
|
|
|
class TestHasKnownPricing:
|
|
def test_known_commercial_model(self):
|
|
assert _has_known_pricing("gpt-4o", provider="openai") is True
|
|
assert _has_known_pricing("anthropic/claude-sonnet-4-20250514") is True
|
|
assert _has_known_pricing("gpt-4.1", provider="openai") is True
|
|
|
|
def test_unknown_custom_model(self):
|
|
assert _has_known_pricing("FP16_Hermes_4.5") is False
|
|
assert _has_known_pricing("my-custom-model") is False
|
|
assert _has_known_pricing("glm-5") is False
|
|
assert _has_known_pricing("") is False
|
|
assert _has_known_pricing(None) is False
|
|
|
|
def test_heuristic_matched_models_are_not_considered_known(self):
|
|
assert _has_known_pricing("some-opus-model") is False
|
|
assert _has_known_pricing("future-sonnet-v2") is False
|
|
|
|
|
|
class TestEstimateCost:
|
|
def test_basic_cost(self):
|
|
cost, status = _estimate_cost(
|
|
"anthropic/claude-sonnet-4-20250514",
|
|
1_000_000,
|
|
1_000_000,
|
|
provider="anthropic",
|
|
)
|
|
assert status == "estimated"
|
|
assert cost == pytest.approx(18.0, abs=0.01)
|
|
|
|
def test_zero_tokens(self):
|
|
cost, status = _estimate_cost("gpt-4o", 0, 0, provider="openai")
|
|
assert status == "estimated"
|
|
assert cost == 0.0
|
|
|
|
def test_cache_aware_usage(self):
|
|
cost, status = _estimate_cost(
|
|
"anthropic/claude-sonnet-4-20250514",
|
|
1000,
|
|
500,
|
|
cache_read_tokens=2000,
|
|
cache_write_tokens=400,
|
|
provider="anthropic",
|
|
)
|
|
assert status == "estimated"
|
|
expected = (1000 * 3.0 + 500 * 15.0 + 2000 * 0.30 + 400 * 3.75) / 1_000_000
|
|
assert cost == pytest.approx(expected, abs=0.0001)
|
|
|
|
|
|
# =========================================================================
|
|
# Format helpers
|
|
# =========================================================================
|
|
|
|
class TestFormatDuration:
|
|
def test_seconds(self):
|
|
assert _format_duration(45) == "45s"
|
|
|
|
|
|
def test_hours_with_minutes(self):
|
|
result = _format_duration(5400) # 1.5 hours
|
|
assert result == "1h 30m"
|
|
|
|
|
|
|
|
|
|
class TestBarChart:
|
|
def test_basic_bars(self):
|
|
bars = _bar_chart([10, 5, 0, 20], max_width=10)
|
|
assert len(bars) == 4
|
|
assert len(bars[3]) == 10 # max value gets full width
|
|
assert len(bars[0]) == 5 # half of max
|
|
assert bars[2] == "" # zero gets empty
|
|
|
|
|
|
def test_all_zeros(self):
|
|
bars = _bar_chart([0, 0, 0], max_width=10)
|
|
assert all(b == "" for b in bars)
|
|
|
|
|
|
|
|
# =========================================================================
|
|
# InsightsEngine — empty DB
|
|
# =========================================================================
|
|
|
|
class TestInsightsEmpty:
|
|
def test_empty_db_returns_empty_report(self, db):
|
|
engine = InsightsEngine(db)
|
|
report = engine.generate(days=30)
|
|
assert report["empty"] is True
|
|
assert report["overview"] == {}
|
|
|
|
def test_empty_db_terminal_format(self, db):
|
|
engine = InsightsEngine(db)
|
|
report = engine.generate(days=30)
|
|
text = engine.format_terminal(report)
|
|
assert "No sessions found" in text
|
|
|
|
def test_empty_db_gateway_format(self, db):
|
|
engine = InsightsEngine(db)
|
|
report = engine.generate(days=30)
|
|
text = engine.format_gateway(report)
|
|
assert "No sessions found" in text
|
|
|
|
|
|
# =========================================================================
|
|
# InsightsEngine — populated DB
|
|
# =========================================================================
|
|
|
|
class TestInsightsPopulated:
|
|
|
|
|
|
def test_overview_token_totals(self, populated_db):
|
|
engine = InsightsEngine(populated_db)
|
|
report = engine.generate(days=30)
|
|
overview = report["overview"]
|
|
|
|
expected_input = 50000 + 20000 + 100000 + 10000
|
|
expected_output = 15000 + 8000 + 40000 + 5000
|
|
assert overview["total_input_tokens"] == expected_input
|
|
assert overview["total_output_tokens"] == expected_output
|
|
assert overview["total_tokens"] == expected_input + expected_output
|
|
|
|
|
|
|
|
|
|
def test_model_breakdown_splits_mid_session_switch(self, db):
|
|
"""A session that switches models mid-flight is split across both
|
|
models in the breakdown, not dumped on the initial model (#51607).
|
|
"""
|
|
now = time.time()
|
|
db.create_session(session_id="sw", source="cli",
|
|
model="deepseek/deepseek-v4-pro")
|
|
# 40k tokens on deepseek, then switch and 50k on opus.
|
|
db.update_token_counts("sw", input_tokens=40000, output_tokens=8000,
|
|
model="deepseek/deepseek-v4-pro",
|
|
billing_provider="deepseek", api_call_count=2)
|
|
db.update_session_model("sw", "anthropic/claude-opus-4.8")
|
|
db.update_token_counts("sw", input_tokens=50000, output_tokens=4000,
|
|
model="anthropic/claude-opus-4.8",
|
|
billing_provider="openrouter", api_call_count=3)
|
|
db._conn.commit()
|
|
|
|
report = InsightsEngine(db).generate(days=30)
|
|
models = {m["model"]: m for m in report["models"]}
|
|
assert "deepseek-v4-pro" in models
|
|
assert "claude-opus-4.8" in models
|
|
# Tokens attributed to the model that actually incurred them.
|
|
assert models["deepseek-v4-pro"]["input_tokens"] == 40000
|
|
assert models["claude-opus-4.8"]["input_tokens"] == 50000
|
|
assert models["claude-opus-4.8"]["api_calls"] == 3
|
|
# The summary row's single model would have hidden one of these.
|
|
assert models["deepseek-v4-pro"]["total_tokens"] == 48000
|
|
assert models["claude-opus-4.8"]["total_tokens"] == 54000
|
|
|
|
|
|
def test_overview_cost_matches_per_model_stored_cost(self, db):
|
|
db.create_session(session_id="cost", source="cli", model="model-a")
|
|
db.update_token_counts(
|
|
"cost", input_tokens=10, model="model-a", billing_provider="custom",
|
|
estimated_cost_usd=1.25, actual_cost_usd=1.0,
|
|
cost_status="estimated", cost_source="provider", api_call_count=1,
|
|
)
|
|
db.update_session_model("cost", "model-b")
|
|
db.update_session_billing_route("cost", provider="custom-b", base_url=None)
|
|
db.update_token_counts(
|
|
"cost", input_tokens=20, model="model-b", billing_provider="custom-b",
|
|
estimated_cost_usd=2.5, actual_cost_usd=2.0,
|
|
cost_status="estimated", cost_source="provider", api_call_count=1,
|
|
)
|
|
|
|
report = InsightsEngine(db).generate(days=30)
|
|
assert sum(m["cost"] for m in report["models"]) == pytest.approx(3.75)
|
|
assert report["overview"]["estimated_cost"] == pytest.approx(3.75)
|
|
assert report["overview"]["actual_cost"] == pytest.approx(3.0)
|
|
|
|
|
|
def test_tool_breakdown(self, populated_db):
|
|
engine = InsightsEngine(populated_db)
|
|
report = engine.generate(days=30)
|
|
tools = report["tools"]
|
|
|
|
tool_names = [t["tool"] for t in tools]
|
|
assert "terminal" in tool_names
|
|
assert "search_files" in tool_names
|
|
assert "read_file" in tool_names
|
|
assert "patch" in tool_names
|
|
assert "web_search" in tool_names
|
|
|
|
# terminal was used 2x in s3
|
|
terminal = next(t for t in tools if t["tool"] == "terminal")
|
|
assert terminal["count"] == 2
|
|
|
|
# Percentages should sum to ~100%
|
|
total_pct = sum(t["percentage"] for t in tools)
|
|
assert total_pct == pytest.approx(100.0, abs=0.1)
|
|
|
|
def test_skill_breakdown(self, populated_db):
|
|
engine = InsightsEngine(populated_db)
|
|
report = engine.generate(days=30)
|
|
skills = report["skills"]
|
|
|
|
assert skills["summary"]["distinct_skills_used"] == 3
|
|
assert skills["summary"]["total_skill_loads"] == 3
|
|
assert skills["summary"]["total_skill_edits"] == 1
|
|
assert skills["summary"]["total_skill_actions"] == 4
|
|
|
|
top_skill = skills["top_skills"][0]
|
|
assert top_skill["skill"] == "github-pr-workflow"
|
|
assert top_skill["view_count"] == 2
|
|
assert top_skill["manage_count"] == 0
|
|
assert top_skill["total_count"] == 2
|
|
assert top_skill["last_used_at"] is not None
|
|
|
|
|
|
def test_activity_patterns(self, populated_db):
|
|
engine = InsightsEngine(populated_db)
|
|
report = engine.generate(days=30)
|
|
activity = report["activity"]
|
|
|
|
assert len(activity["by_day"]) == 7
|
|
assert len(activity["by_hour"]) == 24
|
|
assert activity["active_days"] >= 1
|
|
assert activity["busiest_day"] is not None
|
|
assert activity["busiest_hour"] is not None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# =========================================================================
|
|
# Formatting
|
|
# =========================================================================
|
|
|
|
class TestTerminalFormatting:
|
|
def test_terminal_format_has_sections(self, populated_db):
|
|
engine = InsightsEngine(populated_db)
|
|
report = engine.generate(days=30)
|
|
text = engine.format_terminal(report)
|
|
|
|
assert "Hermes Insights" in text
|
|
assert "Overview" in text
|
|
assert "Models Used" in text
|
|
assert "Top Tools" in text
|
|
assert "Top Skills" in text
|
|
assert "Activity Patterns" in text
|
|
assert "Notable Sessions" in text
|
|
|
|
|
|
|
|
|
|
def test_terminal_format_hides_cost_for_custom_models(self, db):
|
|
"""Cost display is hidden entirely — custom models no longer show 'N/A' either."""
|
|
db.create_session(session_id="s1", source="cli", model="my-custom-model")
|
|
db.update_token_counts("s1", input_tokens=1000, output_tokens=500)
|
|
db._conn.commit()
|
|
|
|
engine = InsightsEngine(db)
|
|
report = engine.generate(days=30)
|
|
text = engine.format_terminal(report)
|
|
|
|
assert "N/A" not in text
|
|
assert "custom/self-hosted" not in text
|
|
assert "Cost" not in text
|
|
|
|
|
|
class TestGatewayFormatting:
|
|
def test_gateway_format_is_shorter(self, populated_db):
|
|
engine = InsightsEngine(populated_db)
|
|
report = engine.generate(days=30)
|
|
terminal_text = engine.format_terminal(report)
|
|
gateway_text = engine.format_gateway(report)
|
|
|
|
assert len(gateway_text) < len(terminal_text)
|
|
|
|
|
|
def test_gateway_format_hides_cost(self, populated_db):
|
|
"""Gateway format omits dollar figures and internal cache details."""
|
|
engine = InsightsEngine(populated_db)
|
|
report = engine.generate(days=30)
|
|
text = engine.format_gateway(report)
|
|
|
|
assert "$" not in text
|
|
assert "cache" not in text.lower()
|
|
|
|
|
|
|
|
# =========================================================================
|
|
# Edge cases
|
|
# =========================================================================
|
|
|
|
class TestEdgeCases:
|
|
|
|
|
|
def test_session_with_no_model(self, db):
|
|
"""Sessions with NULL model should not crash."""
|
|
db.create_session(session_id="s1", source="cli")
|
|
db.update_token_counts("s1", input_tokens=1000, output_tokens=500)
|
|
db._conn.commit()
|
|
|
|
engine = InsightsEngine(db)
|
|
report = engine.generate(days=30)
|
|
assert report["empty"] is False
|
|
|
|
models = report["models"]
|
|
assert len(models) == 1
|
|
assert models[0]["model"] == "unknown"
|
|
assert models[0]["has_pricing"] is False
|
|
|
|
def test_custom_model_shows_zero_cost(self, db):
|
|
"""Custom/self-hosted models should show $0 cost, not fake estimates."""
|
|
db.create_session(session_id="s1", source="cli", model="FP16_Hermes_4.5")
|
|
db.update_token_counts("s1", input_tokens=100000, output_tokens=50000)
|
|
db._conn.commit()
|
|
|
|
engine = InsightsEngine(db)
|
|
report = engine.generate(days=30)
|
|
assert report["overview"]["estimated_cost"] == 0.0
|
|
assert "FP16_Hermes_4.5" in report["overview"]["models_without_pricing"]
|
|
|
|
models = report["models"]
|
|
custom = next(m for m in models if m["model"] == "FP16_Hermes_4.5")
|
|
assert custom["cost"] == 0.0
|
|
assert custom["has_pricing"] is False
|
|
|
|
|
|
|
|
def test_mixed_commercial_and_custom_models(self, db):
|
|
"""Mix of commercial and custom models: only commercial ones get costs."""
|
|
db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4-20250514")
|
|
db.update_token_counts(
|
|
"s1",
|
|
input_tokens=10000,
|
|
output_tokens=5000,
|
|
billing_provider="anthropic",
|
|
)
|
|
db.create_session(session_id="s2", source="cli", model="my-local-llama")
|
|
db.update_token_counts("s2", input_tokens=10000, output_tokens=5000)
|
|
db._conn.commit()
|
|
|
|
engine = InsightsEngine(db)
|
|
report = engine.generate(days=30)
|
|
|
|
# Cost should only come from gpt-4o, not from the custom model
|
|
overview = report["overview"]
|
|
assert overview["estimated_cost"] > 0
|
|
assert "claude-sonnet-4-20250514" in overview["models_with_pricing"] # list now, not set
|
|
assert "my-local-llama" in overview["models_without_pricing"]
|
|
|
|
# Verify individual model entries
|
|
claude = next(m for m in report["models"] if m["model"] == "claude-sonnet-4-20250514")
|
|
assert claude["has_pricing"] is True
|
|
assert claude["cost"] > 0
|
|
|
|
llama = next(m for m in report["models"] if m["model"] == "my-local-llama")
|
|
assert llama["has_pricing"] is False
|
|
assert llama["cost"] == 0.0
|
|
|
|
|
|
|
|
def test_only_one_platform(self, db):
|
|
"""Single-platform usage should still work."""
|
|
db.create_session(session_id="s1", source="cli", model="test")
|
|
db._conn.commit()
|
|
|
|
engine = InsightsEngine(db)
|
|
report = engine.generate(days=30)
|
|
assert len(report["platforms"]) == 1
|
|
assert report["platforms"][0]["platform"] == "cli"
|
|
|
|
# Terminal format should NOT show platform section for single platform
|
|
text = engine.format_terminal(report)
|
|
# (it still shows platforms section if there's only cli and nothing else)
|
|
# Actually the condition is > 1 platforms OR non-cli, so single cli won't show
|
|
|
|
|