From 857b543543ab5faeef5ba851c3878fe289493ad4 Mon Sep 17 00:00:00 2001 From: Arihant Sethia Date: Wed, 15 Apr 2026 06:12:35 +0000 Subject: [PATCH] feat: add skill analytics to the dashboard Expose skill usage in analytics so the dashboard and insights output can show which skills the agent loads and manages over time. This adds skill aggregation to the InsightsEngine by extracting `skill_view` and `skill_manage` calls from assistant tool_calls, computing per-skill totals, and including the results in both terminal and gateway insights formatting. It also extends the dashboard analytics API and Analytics page to render a Top Skills table. Terminology is aligned with the skills docs: - Agent Loaded = `skill_view` events - Agent Managed = `skill_manage` actions Architecture: - agent/insights.py collects and aggregates per-skill usage - hermes_cli/web_server.py exposes `skills` on `/api/analytics/usage` - web/src/lib/api.ts adds analytics skill response types - web/src/pages/AnalyticsPage.tsx renders the Top Skills table - web/src/i18n/{en,zh}.ts updates user-facing labels Tests: - tests/agent/test_insights.py covers skill aggregation and formatting - tests/hermes_cli/test_web_server.py covers analytics API contract including the `skills` payload - verified with `cd web && npm run build` Files changed: - agent/insights.py - hermes_cli/web_server.py - tests/agent/test_insights.py - tests/hermes_cli/test_web_server.py - web/src/i18n/en.ts - web/src/i18n/types.ts - web/src/i18n/zh.ts - web/src/lib/api.ts - web/src/pages/AnalyticsPage.tsx --- agent/insights.py | 162 ++++++++++++++++++++++++++++ hermes_cli/web_server.py | 20 +++- tests/agent/test_insights.py | 52 +++++++++ tests/hermes_cli/test_web_server.py | 83 +++++++++++++- web/src/i18n/en.ts | 5 + web/src/i18n/types.ts | 5 + web/src/i18n/zh.ts | 5 + web/src/lib/api.ts | 20 ++++ web/src/pages/AnalyticsPage.tsx | 53 ++++++++- 9 files changed, 399 insertions(+), 6 deletions(-) diff --git a/agent/insights.py b/agent/insights.py index a0929c912..8972f94a8 100644 --- a/agent/insights.py +++ b/agent/insights.py @@ -124,6 +124,7 @@ class InsightsEngine: # Gather raw data sessions = self._get_sessions(cutoff, source) tool_usage = self._get_tool_usage(cutoff, source) + skill_usage = self._get_skill_usage(cutoff, source) message_stats = self._get_message_stats(cutoff, source) if not sessions: @@ -135,6 +136,15 @@ class InsightsEngine: "models": [], "platforms": [], "tools": [], + "skills": { + "summary": { + "total_skill_loads": 0, + "total_skill_edits": 0, + "total_skill_actions": 0, + "distinct_skills_used": 0, + }, + "top_skills": [], + }, "activity": {}, "top_sessions": [], } @@ -144,6 +154,7 @@ class InsightsEngine: models = self._compute_model_breakdown(sessions) platforms = self._compute_platform_breakdown(sessions) tools = self._compute_tool_breakdown(tool_usage) + skills = self._compute_skill_breakdown(skill_usage) activity = self._compute_activity_patterns(sessions) top_sessions = self._compute_top_sessions(sessions) @@ -156,6 +167,7 @@ class InsightsEngine: "models": models, "platforms": platforms, "tools": tools, + "skills": skills, "activity": activity, "top_sessions": top_sessions, } @@ -284,6 +296,82 @@ class InsightsEngine: for name, count in tool_counts.most_common() ] + def _get_skill_usage(self, cutoff: float, source: str = None) -> List[Dict]: + """Extract per-skill usage from assistant tool calls.""" + skill_counts: Dict[str, Dict[str, Any]] = {} + + if source: + cursor = self._conn.execute( + """SELECT m.tool_calls, m.timestamp + FROM messages m + JOIN sessions s ON s.id = m.session_id + WHERE s.started_at >= ? AND s.source = ? + AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""", + (cutoff, source), + ) + else: + cursor = self._conn.execute( + """SELECT m.tool_calls, m.timestamp + FROM messages m + JOIN sessions s ON s.id = m.session_id + WHERE s.started_at >= ? + AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""", + (cutoff,), + ) + + for row in cursor.fetchall(): + try: + calls = row["tool_calls"] + if isinstance(calls, str): + calls = json.loads(calls) + if not isinstance(calls, list): + continue + except (json.JSONDecodeError, TypeError): + continue + + timestamp = row["timestamp"] + for call in calls: + if not isinstance(call, dict): + continue + func = call.get("function", {}) + tool_name = func.get("name") + if tool_name not in {"skill_view", "skill_manage"}: + continue + + args = func.get("arguments") + if isinstance(args, str): + try: + args = json.loads(args) + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(args, dict): + continue + + skill_name = args.get("name") + if not isinstance(skill_name, str) or not skill_name.strip(): + continue + + entry = skill_counts.setdefault( + skill_name, + { + "skill": skill_name, + "view_count": 0, + "manage_count": 0, + "last_used_at": None, + }, + ) + if tool_name == "skill_view": + entry["view_count"] += 1 + else: + entry["manage_count"] += 1 + + if timestamp is not None and ( + entry["last_used_at"] is None or timestamp > entry["last_used_at"] + ): + entry["last_used_at"] = timestamp + + return list(skill_counts.values()) + def _get_message_stats(self, cutoff: float, source: str = None) -> Dict: """Get aggregate message statistics.""" if source: @@ -475,6 +563,46 @@ class InsightsEngine: }) return result + def _compute_skill_breakdown(self, skill_usage: List[Dict]) -> Dict[str, Any]: + """Process per-skill usage into summary + ranked list.""" + total_skill_loads = sum(s["view_count"] for s in skill_usage) if skill_usage else 0 + total_skill_edits = sum(s["manage_count"] for s in skill_usage) if skill_usage else 0 + total_skill_actions = total_skill_loads + total_skill_edits + + top_skills = [] + for skill in skill_usage: + total_count = skill["view_count"] + skill["manage_count"] + percentage = (total_count / total_skill_actions * 100) if total_skill_actions else 0 + top_skills.append({ + "skill": skill["skill"], + "view_count": skill["view_count"], + "manage_count": skill["manage_count"], + "total_count": total_count, + "percentage": percentage, + "last_used_at": skill.get("last_used_at"), + }) + + top_skills.sort( + key=lambda s: ( + s["total_count"], + s["view_count"], + s["manage_count"], + s["last_used_at"] or 0, + s["skill"], + ), + reverse=True, + ) + + return { + "summary": { + "total_skill_loads": total_skill_loads, + "total_skill_edits": total_skill_edits, + "total_skill_actions": total_skill_actions, + "distinct_skills_used": len(skill_usage), + }, + "top_skills": top_skills, + } + def _compute_activity_patterns(self, sessions: List[Dict]) -> Dict: """Analyze activity patterns by day of week and hour.""" day_counts = Counter() # 0=Monday ... 6=Sunday @@ -682,6 +810,28 @@ class InsightsEngine: lines.append(f" ... and {len(report['tools']) - 15} more tools") lines.append("") + # Skill usage + skills = report.get("skills", {}) + top_skills = skills.get("top_skills", []) + if top_skills: + lines.append(" 🧠 Top Skills") + lines.append(" " + "─" * 56) + lines.append(f" {'Skill':<28} {'Loads':>7} {'Edits':>7} {'Last used':>11}") + for skill in top_skills[:10]: + last_used = "—" + if skill.get("last_used_at"): + last_used = datetime.fromtimestamp(skill["last_used_at"]).strftime("%b %d") + lines.append( + f" {skill['skill'][:28]:<28} {skill['view_count']:>7,} {skill['manage_count']:>7,} {last_used:>11}" + ) + summary = skills.get("summary", {}) + lines.append( + f" Distinct skills: {summary.get('distinct_skills_used', 0)} " + f"Loads: {summary.get('total_skill_loads', 0):,} " + f"Edits: {summary.get('total_skill_edits', 0):,}" + ) + lines.append("") + # Activity patterns act = report.get("activity", {}) if act.get("by_day"): @@ -774,6 +924,18 @@ class InsightsEngine: lines.append(f" {t['tool']} — {t['count']:,} calls ({t['percentage']:.1f}%)") lines.append("") + skills = report.get("skills", {}) + if skills.get("top_skills"): + lines.append("**🧠 Top Skills:**") + for skill in skills["top_skills"][:5]: + suffix = "" + if skill.get("last_used_at"): + suffix = f", last used {datetime.fromtimestamp(skill['last_used_at']).strftime('%b %d')}" + lines.append( + f" {skill['skill']} — {skill['view_count']:,} loads, {skill['manage_count']:,} edits{suffix}" + ) + lines.append("") + # Activity summary act = report.get("activity", {}) if act.get("busiest_day") and act.get("busiest_hour"): diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 22265faa5..f18afbf86 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -1977,6 +1977,8 @@ async def update_config_raw(body: RawConfigUpdate): @app.get("/api/analytics/usage") async def get_usage_analytics(days: int = 30): from hermes_state import SessionDB + from agent.insights import InsightsEngine + db = SessionDB() try: cutoff = time.time() - (days * 86400) @@ -2016,8 +2018,24 @@ async def get_usage_analytics(days: int = 30): FROM sessions WHERE started_at > ? """, (cutoff,)) totals = dict(cur3.fetchone()) + insights_report = InsightsEngine(db).generate(days=days) + skills = insights_report.get("skills", { + "summary": { + "total_skill_loads": 0, + "total_skill_edits": 0, + "total_skill_actions": 0, + "distinct_skills_used": 0, + }, + "top_skills": [], + }) - return {"daily": daily, "by_model": by_model, "totals": totals, "period_days": days} + return { + "daily": daily, + "by_model": by_model, + "totals": totals, + "period_days": days, + "skills": skills, + } finally: db.close() diff --git a/tests/agent/test_insights.py b/tests/agent/test_insights.py index 885e34fec..7ca8a9792 100644 --- a/tests/agent/test_insights.py +++ b/tests/agent/test_insights.py @@ -51,6 +51,12 @@ def populated_db(db): db.append_message("s1", role="assistant", content="I found the bug. Let me fix it.", tool_calls=[{"function": {"name": "patch"}}]) db.append_message("s1", role="tool", content="patched successfully", tool_name="patch") + db.append_message( + "s1", + role="assistant", + content="Let me load the PR workflow skill.", + tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}}], + ) db.append_message("s1", role="user", content="Thanks!") db.append_message("s1", role="assistant", content="You're welcome!") @@ -88,6 +94,12 @@ def populated_db(db): db.append_message("s3", role="assistant", content="And search files", tool_calls=[{"function": {"name": "search_files"}}]) db.append_message("s3", role="tool", content="found stuff", tool_name="search_files") + db.append_message( + "s3", + role="assistant", + content="Load the debugging skill.", + tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"systematic-debugging"}'}}], + ) # Session 4: Discord, same model as s1, ended, 1 day ago db.create_session( @@ -100,6 +112,15 @@ def populated_db(db): db.update_token_counts("s4", input_tokens=10000, output_tokens=5000) db.append_message("s4", role="user", content="Quick question") db.append_message("s4", role="assistant", content="Sure, go ahead") + db.append_message( + "s4", + role="assistant", + content="Load and update GitHub skills.", + tool_calls=[ + {"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}}, + {"function": {"name": "skill_manage", "arguments": '{"name":"github-code-review"}'}}, + ], + ) # Session 5: Old session, 45 days ago (should be excluded from 30-day window) db.create_session( @@ -332,6 +353,35 @@ class TestInsightsPopulated: total_pct = sum(t["percentage"] for t in tools) assert total_pct == pytest.approx(100.0, abs=0.1) + def test_skill_breakdown(self, populated_db): + engine = InsightsEngine(populated_db) + report = engine.generate(days=30) + skills = report["skills"] + + assert skills["summary"]["distinct_skills_used"] == 3 + assert skills["summary"]["total_skill_loads"] == 3 + assert skills["summary"]["total_skill_edits"] == 1 + assert skills["summary"]["total_skill_actions"] == 4 + + top_skill = skills["top_skills"][0] + assert top_skill["skill"] == "github-pr-workflow" + assert top_skill["view_count"] == 2 + assert top_skill["manage_count"] == 0 + assert top_skill["total_count"] == 2 + assert top_skill["last_used_at"] is not None + + def test_skill_breakdown_respects_days_filter(self, populated_db): + engine = InsightsEngine(populated_db) + report = engine.generate(days=3) + skills = report["skills"] + + assert skills["summary"]["distinct_skills_used"] == 2 + assert skills["summary"]["total_skill_loads"] == 2 + assert skills["summary"]["total_skill_edits"] == 1 + + skill_names = [s["skill"] for s in skills["top_skills"]] + assert "systematic-debugging" not in skill_names + def test_activity_patterns(self, populated_db): engine = InsightsEngine(populated_db) report = engine.generate(days=30) @@ -401,6 +451,7 @@ class TestTerminalFormatting: assert "Overview" in text assert "Models Used" in text assert "Top Tools" in text + assert "Top Skills" in text assert "Activity Patterns" in text assert "Notable Sessions" in text @@ -467,6 +518,7 @@ class TestGatewayFormatting: text = engine.format_gateway(report) assert "$" in text + assert "Top Skills" in text assert "Est. cost" in text def test_gateway_format_shows_models(self, populated_db): diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 365e3d0fe..fa7ce62b2 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -101,14 +101,19 @@ class TestWebServerEndpoints: """Test the FastAPI REST endpoints using Starlette TestClient.""" @pytest.fixture(autouse=True) - def _setup_test_client(self): - """Create a TestClient — import is deferred to avoid requiring fastapi.""" + def _setup_test_client(self, monkeypatch, _isolate_hermes_home): + """Create a TestClient and isolate the state DB under the test HERMES_HOME.""" try: from starlette.testclient import TestClient except ImportError: pytest.skip("fastapi/starlette not installed") + import hermes_state + from hermes_constants import get_hermes_home from hermes_cli.web_server import app, _SESSION_TOKEN + + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db") + self.client = TestClient(app) self.client.headers["Authorization"] = f"Bearer {_SESSION_TOKEN}" @@ -511,12 +516,18 @@ class TestNewEndpoints: """Tests for session detail, logs, cron, skills, tools, raw config, analytics.""" @pytest.fixture(autouse=True) - def _setup(self): + def _setup(self, monkeypatch, _isolate_hermes_home): try: from starlette.testclient import TestClient except ImportError: pytest.skip("fastapi/starlette not installed") + + import hermes_state + from hermes_constants import get_hermes_home from hermes_cli.web_server import app, _SESSION_TOKEN + + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db") + self.client = TestClient(app) self.client.headers["Authorization"] = f"Bearer {_SESSION_TOKEN}" @@ -692,8 +703,74 @@ class TestNewEndpoints: assert "daily" in data assert "by_model" in data assert "totals" in data + assert "skills" in data assert isinstance(data["daily"], list) assert "total_sessions" in data["totals"] + assert data["skills"] == { + "summary": { + "total_skill_loads": 0, + "total_skill_edits": 0, + "total_skill_actions": 0, + "distinct_skills_used": 0, + }, + "top_skills": [], + } + + def test_analytics_usage_includes_skill_breakdown(self): + from hermes_state import SessionDB + + db = SessionDB() + try: + db.create_session( + session_id="skills-analytics-test", + source="cli", + model="anthropic/claude-sonnet-4", + ) + db.update_token_counts( + "skills-analytics-test", + input_tokens=120, + output_tokens=45, + ) + db.append_message( + "skills-analytics-test", + role="assistant", + content="Loading and updating skills.", + tool_calls=[ + { + "function": { + "name": "skill_view", + "arguments": '{"name":"github-pr-workflow"}', + } + }, + { + "function": { + "name": "skill_manage", + "arguments": '{"name":"github-code-review"}', + } + }, + ], + ) + finally: + db.close() + + resp = self.client.get("/api/analytics/usage?days=7") + assert resp.status_code == 200 + + data = resp.json() + assert data["skills"]["summary"] == { + "total_skill_loads": 1, + "total_skill_edits": 1, + "total_skill_actions": 2, + "distinct_skills_used": 2, + } + assert len(data["skills"]["top_skills"]) == 2 + + top_skill = data["skills"]["top_skills"][0] + assert top_skill["skill"] == "github-pr-workflow" + assert top_skill["view_count"] == 1 + assert top_skill["manage_count"] == 0 + assert top_skill["total_count"] == 1 + assert top_skill["last_used_at"] is not None def test_session_token_endpoint_removed(self): """GET /api/auth/session-token no longer exists.""" diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 3bf693f21..b15be08a4 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -115,6 +115,11 @@ export const en: Translations = { dailyTokenUsage: "Daily Token Usage", dailyBreakdown: "Daily Breakdown", perModelBreakdown: "Per-Model Breakdown", + topSkills: "Top Skills", + skill: "Skill", + loads: "Agent Loaded", + edits: "Agent Managed", + lastUsed: "Last Used", input: "Input", output: "Output", total: "Total", diff --git a/web/src/i18n/types.ts b/web/src/i18n/types.ts index 34813c68f..3996fd1f0 100644 --- a/web/src/i18n/types.ts +++ b/web/src/i18n/types.ts @@ -120,6 +120,11 @@ export interface Translations { dailyTokenUsage: string; dailyBreakdown: string; perModelBreakdown: string; + topSkills: string; + skill: string; + loads: string; + edits: string; + lastUsed: string; input: string; output: string; total: string; diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 18cb3ee38..c4e334a88 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -115,6 +115,11 @@ export const zh: Translations = { dailyTokenUsage: "每日 Token 用量", dailyBreakdown: "每日明细", perModelBreakdown: "模型用量明细", + topSkills: "常用技能", + skill: "技能", + loads: "代理加载", + edits: "代理管理", + lastUsed: "最近使用", input: "输入", output: "输出", total: "总计", diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index e61043993..b82c7808c 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -283,6 +283,22 @@ export interface AnalyticsModelEntry { sessions: number; } +export interface AnalyticsSkillEntry { + skill: string; + view_count: number; + manage_count: number; + total_count: number; + percentage: number; + last_used_at: number | null; +} + +export interface AnalyticsSkillsSummary { + total_skill_loads: number; + total_skill_edits: number; + total_skill_actions: number; + distinct_skills_used: number; +} + export interface AnalyticsResponse { daily: AnalyticsDailyEntry[]; by_model: AnalyticsModelEntry[]; @@ -295,6 +311,10 @@ export interface AnalyticsResponse { total_actual_cost: number; total_sessions: number; }; + skills: { + summary: AnalyticsSkillsSummary; + top_skills: AnalyticsSkillEntry[]; + }; } export interface CronJob { diff --git a/web/src/pages/AnalyticsPage.tsx b/web/src/pages/AnalyticsPage.tsx index 2f947cbb6..c9efd70ac 100644 --- a/web/src/pages/AnalyticsPage.tsx +++ b/web/src/pages/AnalyticsPage.tsx @@ -1,12 +1,14 @@ import { useEffect, useState, useCallback } from "react"; import { BarChart3, + Brain, Cpu, Hash, TrendingUp, } from "lucide-react"; import { api } from "@/lib/api"; -import type { AnalyticsResponse, AnalyticsDailyEntry, AnalyticsModelEntry } from "@/lib/api"; +import type { AnalyticsResponse, AnalyticsDailyEntry, AnalyticsModelEntry, AnalyticsSkillEntry } from "@/lib/api"; +import { timeAgo } from "@/lib/utils"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; import { useI18n } from "@/i18n"; @@ -227,6 +229,52 @@ function ModelTable({ models }: { models: AnalyticsModelEntry[] }) { ); } +function SkillTable({ skills }: { skills: AnalyticsSkillEntry[] }) { + const { t } = useI18n(); + if (skills.length === 0) return null; + + return ( + + +
+ + {t.analytics.topSkills} +
+ + +
+ + + + + + + + + + + + {skills.map((skill) => ( + + + + + + + + ))} + +
{t.analytics.skill}{t.analytics.loads}{t.analytics.edits}{t.analytics.total}{t.analytics.lastUsed}
+ {skill.skill} + {skill.view_count}{skill.manage_count}{skill.total_count} + {skill.last_used_at ? timeAgo(skill.last_used_at) : "—"} +
+
+ + + ); +} + export default function AnalyticsPage() { const [days, setDays] = useState(30); const [data, setData] = useState(null); @@ -310,10 +358,11 @@ export default function AnalyticsPage() { {/* Tables */} + )} - {data && data.daily.length === 0 && data.by_model.length === 0 && ( + {data && data.daily.length === 0 && data.by_model.length === 0 && data.skills.top_skills.length === 0 && (