diff --git a/agent/insights.py b/agent/insights.py index a0929c912..8972f94a8 100644 --- a/agent/insights.py +++ b/agent/insights.py @@ -124,6 +124,7 @@ class InsightsEngine: # Gather raw data sessions = self._get_sessions(cutoff, source) tool_usage = self._get_tool_usage(cutoff, source) + skill_usage = self._get_skill_usage(cutoff, source) message_stats = self._get_message_stats(cutoff, source) if not sessions: @@ -135,6 +136,15 @@ class InsightsEngine: "models": [], "platforms": [], "tools": [], + "skills": { + "summary": { + "total_skill_loads": 0, + "total_skill_edits": 0, + "total_skill_actions": 0, + "distinct_skills_used": 0, + }, + "top_skills": [], + }, "activity": {}, "top_sessions": [], } @@ -144,6 +154,7 @@ class InsightsEngine: models = self._compute_model_breakdown(sessions) platforms = self._compute_platform_breakdown(sessions) tools = self._compute_tool_breakdown(tool_usage) + skills = self._compute_skill_breakdown(skill_usage) activity = self._compute_activity_patterns(sessions) top_sessions = self._compute_top_sessions(sessions) @@ -156,6 +167,7 @@ class InsightsEngine: "models": models, "platforms": platforms, "tools": tools, + "skills": skills, "activity": activity, "top_sessions": top_sessions, } @@ -284,6 +296,82 @@ class InsightsEngine: for name, count in tool_counts.most_common() ] + def _get_skill_usage(self, cutoff: float, source: str = None) -> List[Dict]: + """Extract per-skill usage from assistant tool calls.""" + skill_counts: Dict[str, Dict[str, Any]] = {} + + if source: + cursor = self._conn.execute( + """SELECT m.tool_calls, m.timestamp + FROM messages m + JOIN sessions s ON s.id = m.session_id + WHERE s.started_at >= ? AND s.source = ? + AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""", + (cutoff, source), + ) + else: + cursor = self._conn.execute( + """SELECT m.tool_calls, m.timestamp + FROM messages m + JOIN sessions s ON s.id = m.session_id + WHERE s.started_at >= ? + AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""", + (cutoff,), + ) + + for row in cursor.fetchall(): + try: + calls = row["tool_calls"] + if isinstance(calls, str): + calls = json.loads(calls) + if not isinstance(calls, list): + continue + except (json.JSONDecodeError, TypeError): + continue + + timestamp = row["timestamp"] + for call in calls: + if not isinstance(call, dict): + continue + func = call.get("function", {}) + tool_name = func.get("name") + if tool_name not in {"skill_view", "skill_manage"}: + continue + + args = func.get("arguments") + if isinstance(args, str): + try: + args = json.loads(args) + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(args, dict): + continue + + skill_name = args.get("name") + if not isinstance(skill_name, str) or not skill_name.strip(): + continue + + entry = skill_counts.setdefault( + skill_name, + { + "skill": skill_name, + "view_count": 0, + "manage_count": 0, + "last_used_at": None, + }, + ) + if tool_name == "skill_view": + entry["view_count"] += 1 + else: + entry["manage_count"] += 1 + + if timestamp is not None and ( + entry["last_used_at"] is None or timestamp > entry["last_used_at"] + ): + entry["last_used_at"] = timestamp + + return list(skill_counts.values()) + def _get_message_stats(self, cutoff: float, source: str = None) -> Dict: """Get aggregate message statistics.""" if source: @@ -475,6 +563,46 @@ class InsightsEngine: }) return result + def _compute_skill_breakdown(self, skill_usage: List[Dict]) -> Dict[str, Any]: + """Process per-skill usage into summary + ranked list.""" + total_skill_loads = sum(s["view_count"] for s in skill_usage) if skill_usage else 0 + total_skill_edits = sum(s["manage_count"] for s in skill_usage) if skill_usage else 0 + total_skill_actions = total_skill_loads + total_skill_edits + + top_skills = [] + for skill in skill_usage: + total_count = skill["view_count"] + skill["manage_count"] + percentage = (total_count / total_skill_actions * 100) if total_skill_actions else 0 + top_skills.append({ + "skill": skill["skill"], + "view_count": skill["view_count"], + "manage_count": skill["manage_count"], + "total_count": total_count, + "percentage": percentage, + "last_used_at": skill.get("last_used_at"), + }) + + top_skills.sort( + key=lambda s: ( + s["total_count"], + s["view_count"], + s["manage_count"], + s["last_used_at"] or 0, + s["skill"], + ), + reverse=True, + ) + + return { + "summary": { + "total_skill_loads": total_skill_loads, + "total_skill_edits": total_skill_edits, + "total_skill_actions": total_skill_actions, + "distinct_skills_used": len(skill_usage), + }, + "top_skills": top_skills, + } + def _compute_activity_patterns(self, sessions: List[Dict]) -> Dict: """Analyze activity patterns by day of week and hour.""" day_counts = Counter() # 0=Monday ... 6=Sunday @@ -682,6 +810,28 @@ class InsightsEngine: lines.append(f" ... and {len(report['tools']) - 15} more tools") lines.append("") + # Skill usage + skills = report.get("skills", {}) + top_skills = skills.get("top_skills", []) + if top_skills: + lines.append(" 🧠 Top Skills") + lines.append(" " + "─" * 56) + lines.append(f" {'Skill':<28} {'Loads':>7} {'Edits':>7} {'Last used':>11}") + for skill in top_skills[:10]: + last_used = "—" + if skill.get("last_used_at"): + last_used = datetime.fromtimestamp(skill["last_used_at"]).strftime("%b %d") + lines.append( + f" {skill['skill'][:28]:<28} {skill['view_count']:>7,} {skill['manage_count']:>7,} {last_used:>11}" + ) + summary = skills.get("summary", {}) + lines.append( + f" Distinct skills: {summary.get('distinct_skills_used', 0)} " + f"Loads: {summary.get('total_skill_loads', 0):,} " + f"Edits: {summary.get('total_skill_edits', 0):,}" + ) + lines.append("") + # Activity patterns act = report.get("activity", {}) if act.get("by_day"): @@ -774,6 +924,18 @@ class InsightsEngine: lines.append(f" {t['tool']} — {t['count']:,} calls ({t['percentage']:.1f}%)") lines.append("") + skills = report.get("skills", {}) + if skills.get("top_skills"): + lines.append("**🧠 Top Skills:**") + for skill in skills["top_skills"][:5]: + suffix = "" + if skill.get("last_used_at"): + suffix = f", last used {datetime.fromtimestamp(skill['last_used_at']).strftime('%b %d')}" + lines.append( + f" {skill['skill']} — {skill['view_count']:,} loads, {skill['manage_count']:,} edits{suffix}" + ) + lines.append("") + # Activity summary act = report.get("activity", {}) if act.get("busiest_day") and act.get("busiest_hour"): diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 22265faa5..f18afbf86 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -1977,6 +1977,8 @@ async def update_config_raw(body: RawConfigUpdate): @app.get("/api/analytics/usage") async def get_usage_analytics(days: int = 30): from hermes_state import SessionDB + from agent.insights import InsightsEngine + db = SessionDB() try: cutoff = time.time() - (days * 86400) @@ -2016,8 +2018,24 @@ async def get_usage_analytics(days: int = 30): FROM sessions WHERE started_at > ? """, (cutoff,)) totals = dict(cur3.fetchone()) + insights_report = InsightsEngine(db).generate(days=days) + skills = insights_report.get("skills", { + "summary": { + "total_skill_loads": 0, + "total_skill_edits": 0, + "total_skill_actions": 0, + "distinct_skills_used": 0, + }, + "top_skills": [], + }) - return {"daily": daily, "by_model": by_model, "totals": totals, "period_days": days} + return { + "daily": daily, + "by_model": by_model, + "totals": totals, + "period_days": days, + "skills": skills, + } finally: db.close() diff --git a/tests/agent/test_insights.py b/tests/agent/test_insights.py index 885e34fec..7ca8a9792 100644 --- a/tests/agent/test_insights.py +++ b/tests/agent/test_insights.py @@ -51,6 +51,12 @@ def populated_db(db): db.append_message("s1", role="assistant", content="I found the bug. Let me fix it.", tool_calls=[{"function": {"name": "patch"}}]) db.append_message("s1", role="tool", content="patched successfully", tool_name="patch") + db.append_message( + "s1", + role="assistant", + content="Let me load the PR workflow skill.", + tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}}], + ) db.append_message("s1", role="user", content="Thanks!") db.append_message("s1", role="assistant", content="You're welcome!") @@ -88,6 +94,12 @@ def populated_db(db): db.append_message("s3", role="assistant", content="And search files", tool_calls=[{"function": {"name": "search_files"}}]) db.append_message("s3", role="tool", content="found stuff", tool_name="search_files") + db.append_message( + "s3", + role="assistant", + content="Load the debugging skill.", + tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"systematic-debugging"}'}}], + ) # Session 4: Discord, same model as s1, ended, 1 day ago db.create_session( @@ -100,6 +112,15 @@ def populated_db(db): db.update_token_counts("s4", input_tokens=10000, output_tokens=5000) db.append_message("s4", role="user", content="Quick question") db.append_message("s4", role="assistant", content="Sure, go ahead") + db.append_message( + "s4", + role="assistant", + content="Load and update GitHub skills.", + tool_calls=[ + {"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}}, + {"function": {"name": "skill_manage", "arguments": '{"name":"github-code-review"}'}}, + ], + ) # Session 5: Old session, 45 days ago (should be excluded from 30-day window) db.create_session( @@ -332,6 +353,35 @@ class TestInsightsPopulated: total_pct = sum(t["percentage"] for t in tools) assert total_pct == pytest.approx(100.0, abs=0.1) + def test_skill_breakdown(self, populated_db): + engine = InsightsEngine(populated_db) + report = engine.generate(days=30) + skills = report["skills"] + + assert skills["summary"]["distinct_skills_used"] == 3 + assert skills["summary"]["total_skill_loads"] == 3 + assert skills["summary"]["total_skill_edits"] == 1 + assert skills["summary"]["total_skill_actions"] == 4 + + top_skill = skills["top_skills"][0] + assert top_skill["skill"] == "github-pr-workflow" + assert top_skill["view_count"] == 2 + assert top_skill["manage_count"] == 0 + assert top_skill["total_count"] == 2 + assert top_skill["last_used_at"] is not None + + def test_skill_breakdown_respects_days_filter(self, populated_db): + engine = InsightsEngine(populated_db) + report = engine.generate(days=3) + skills = report["skills"] + + assert skills["summary"]["distinct_skills_used"] == 2 + assert skills["summary"]["total_skill_loads"] == 2 + assert skills["summary"]["total_skill_edits"] == 1 + + skill_names = [s["skill"] for s in skills["top_skills"]] + assert "systematic-debugging" not in skill_names + def test_activity_patterns(self, populated_db): engine = InsightsEngine(populated_db) report = engine.generate(days=30) @@ -401,6 +451,7 @@ class TestTerminalFormatting: assert "Overview" in text assert "Models Used" in text assert "Top Tools" in text + assert "Top Skills" in text assert "Activity Patterns" in text assert "Notable Sessions" in text @@ -467,6 +518,7 @@ class TestGatewayFormatting: text = engine.format_gateway(report) assert "$" in text + assert "Top Skills" in text assert "Est. cost" in text def test_gateway_format_shows_models(self, populated_db): diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 365e3d0fe..fa7ce62b2 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -101,14 +101,19 @@ class TestWebServerEndpoints: """Test the FastAPI REST endpoints using Starlette TestClient.""" @pytest.fixture(autouse=True) - def _setup_test_client(self): - """Create a TestClient — import is deferred to avoid requiring fastapi.""" + def _setup_test_client(self, monkeypatch, _isolate_hermes_home): + """Create a TestClient and isolate the state DB under the test HERMES_HOME.""" try: from starlette.testclient import TestClient except ImportError: pytest.skip("fastapi/starlette not installed") + import hermes_state + from hermes_constants import get_hermes_home from hermes_cli.web_server import app, _SESSION_TOKEN + + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db") + self.client = TestClient(app) self.client.headers["Authorization"] = f"Bearer {_SESSION_TOKEN}" @@ -511,12 +516,18 @@ class TestNewEndpoints: """Tests for session detail, logs, cron, skills, tools, raw config, analytics.""" @pytest.fixture(autouse=True) - def _setup(self): + def _setup(self, monkeypatch, _isolate_hermes_home): try: from starlette.testclient import TestClient except ImportError: pytest.skip("fastapi/starlette not installed") + + import hermes_state + from hermes_constants import get_hermes_home from hermes_cli.web_server import app, _SESSION_TOKEN + + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db") + self.client = TestClient(app) self.client.headers["Authorization"] = f"Bearer {_SESSION_TOKEN}" @@ -692,8 +703,74 @@ class TestNewEndpoints: assert "daily" in data assert "by_model" in data assert "totals" in data + assert "skills" in data assert isinstance(data["daily"], list) assert "total_sessions" in data["totals"] + assert data["skills"] == { + "summary": { + "total_skill_loads": 0, + "total_skill_edits": 0, + "total_skill_actions": 0, + "distinct_skills_used": 0, + }, + "top_skills": [], + } + + def test_analytics_usage_includes_skill_breakdown(self): + from hermes_state import SessionDB + + db = SessionDB() + try: + db.create_session( + session_id="skills-analytics-test", + source="cli", + model="anthropic/claude-sonnet-4", + ) + db.update_token_counts( + "skills-analytics-test", + input_tokens=120, + output_tokens=45, + ) + db.append_message( + "skills-analytics-test", + role="assistant", + content="Loading and updating skills.", + tool_calls=[ + { + "function": { + "name": "skill_view", + "arguments": '{"name":"github-pr-workflow"}', + } + }, + { + "function": { + "name": "skill_manage", + "arguments": '{"name":"github-code-review"}', + } + }, + ], + ) + finally: + db.close() + + resp = self.client.get("/api/analytics/usage?days=7") + assert resp.status_code == 200 + + data = resp.json() + assert data["skills"]["summary"] == { + "total_skill_loads": 1, + "total_skill_edits": 1, + "total_skill_actions": 2, + "distinct_skills_used": 2, + } + assert len(data["skills"]["top_skills"]) == 2 + + top_skill = data["skills"]["top_skills"][0] + assert top_skill["skill"] == "github-pr-workflow" + assert top_skill["view_count"] == 1 + assert top_skill["manage_count"] == 0 + assert top_skill["total_count"] == 1 + assert top_skill["last_used_at"] is not None def test_session_token_endpoint_removed(self): """GET /api/auth/session-token no longer exists.""" diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 3bf693f21..b15be08a4 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -115,6 +115,11 @@ export const en: Translations = { dailyTokenUsage: "Daily Token Usage", dailyBreakdown: "Daily Breakdown", perModelBreakdown: "Per-Model Breakdown", + topSkills: "Top Skills", + skill: "Skill", + loads: "Agent Loaded", + edits: "Agent Managed", + lastUsed: "Last Used", input: "Input", output: "Output", total: "Total", diff --git a/web/src/i18n/types.ts b/web/src/i18n/types.ts index 34813c68f..3996fd1f0 100644 --- a/web/src/i18n/types.ts +++ b/web/src/i18n/types.ts @@ -120,6 +120,11 @@ export interface Translations { dailyTokenUsage: string; dailyBreakdown: string; perModelBreakdown: string; + topSkills: string; + skill: string; + loads: string; + edits: string; + lastUsed: string; input: string; output: string; total: string; diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 18cb3ee38..c4e334a88 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -115,6 +115,11 @@ export const zh: Translations = { dailyTokenUsage: "每日 Token 用量", dailyBreakdown: "每日明细", perModelBreakdown: "模型用量明细", + topSkills: "常用技能", + skill: "技能", + loads: "代理加载", + edits: "代理管理", + lastUsed: "最近使用", input: "输入", output: "输出", total: "总计", diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index e61043993..b82c7808c 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -283,6 +283,22 @@ export interface AnalyticsModelEntry { sessions: number; } +export interface AnalyticsSkillEntry { + skill: string; + view_count: number; + manage_count: number; + total_count: number; + percentage: number; + last_used_at: number | null; +} + +export interface AnalyticsSkillsSummary { + total_skill_loads: number; + total_skill_edits: number; + total_skill_actions: number; + distinct_skills_used: number; +} + export interface AnalyticsResponse { daily: AnalyticsDailyEntry[]; by_model: AnalyticsModelEntry[]; @@ -295,6 +311,10 @@ export interface AnalyticsResponse { total_actual_cost: number; total_sessions: number; }; + skills: { + summary: AnalyticsSkillsSummary; + top_skills: AnalyticsSkillEntry[]; + }; } export interface CronJob { diff --git a/web/src/pages/AnalyticsPage.tsx b/web/src/pages/AnalyticsPage.tsx index 2f947cbb6..c9efd70ac 100644 --- a/web/src/pages/AnalyticsPage.tsx +++ b/web/src/pages/AnalyticsPage.tsx @@ -1,12 +1,14 @@ import { useEffect, useState, useCallback } from "react"; import { BarChart3, + Brain, Cpu, Hash, TrendingUp, } from "lucide-react"; import { api } from "@/lib/api"; -import type { AnalyticsResponse, AnalyticsDailyEntry, AnalyticsModelEntry } from "@/lib/api"; +import type { AnalyticsResponse, AnalyticsDailyEntry, AnalyticsModelEntry, AnalyticsSkillEntry } from "@/lib/api"; +import { timeAgo } from "@/lib/utils"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; import { useI18n } from "@/i18n"; @@ -227,6 +229,52 @@ function ModelTable({ models }: { models: AnalyticsModelEntry[] }) { ); } +function SkillTable({ skills }: { skills: AnalyticsSkillEntry[] }) { + const { t } = useI18n(); + if (skills.length === 0) return null; + + return ( + + +
+ + {t.analytics.topSkills} +
+ + +
+ + + + + + + + + + + + {skills.map((skill) => ( + + + + + + + + ))} + +
{t.analytics.skill}{t.analytics.loads}{t.analytics.edits}{t.analytics.total}{t.analytics.lastUsed}
+ {skill.skill} + {skill.view_count}{skill.manage_count}{skill.total_count} + {skill.last_used_at ? timeAgo(skill.last_used_at) : "—"} +
+
+ + + ); +} + export default function AnalyticsPage() { const [days, setDays] = useState(30); const [data, setData] = useState(null); @@ -310,10 +358,11 @@ export default function AnalyticsPage() { {/* Tables */} + )} - {data && data.daily.length === 0 && data.by_model.length === 0 && ( + {data && data.daily.length === 0 && data.by_model.length === 0 && data.skills.top_skills.length === 0 && (