feat: add skill analytics to the dashboard

Expose skill usage in analytics so the dashboard and insights output can
show which skills the agent loads and manages over time.

This adds skill aggregation to the InsightsEngine by extracting
`skill_view` and `skill_manage` calls from assistant tool_calls,
computing per-skill totals, and including the results in both terminal
and gateway insights formatting. It also extends the dashboard analytics
API and Analytics page to render a Top Skills table.

Terminology is aligned with the skills docs:
  - Agent Loaded = `skill_view` events
  - Agent Managed = `skill_manage` actions

Architecture:
  - agent/insights.py collects and aggregates per-skill usage
  - hermes_cli/web_server.py exposes `skills` on `/api/analytics/usage`
  - web/src/lib/api.ts adds analytics skill response types
  - web/src/pages/AnalyticsPage.tsx renders the Top Skills table
  - web/src/i18n/{en,zh}.ts updates user-facing labels

Tests:
  - tests/agent/test_insights.py covers skill aggregation and formatting
  - tests/hermes_cli/test_web_server.py covers analytics API contract
    including the `skills` payload
  - verified with `cd web && npm run build`

Files changed:
  - agent/insights.py
  - hermes_cli/web_server.py
  - tests/agent/test_insights.py
  - tests/hermes_cli/test_web_server.py
  - web/src/i18n/en.ts
  - web/src/i18n/types.ts
  - web/src/i18n/zh.ts
  - web/src/lib/api.ts
  - web/src/pages/AnalyticsPage.tsx
This commit is contained in:
Arihant Sethia 2026-04-15 06:12:35 +00:00
parent 1525624904
commit 857b543543
9 changed files with 399 additions and 6 deletions

View file

@ -124,6 +124,7 @@ class InsightsEngine:
# Gather raw data
sessions = self._get_sessions(cutoff, source)
tool_usage = self._get_tool_usage(cutoff, source)
skill_usage = self._get_skill_usage(cutoff, source)
message_stats = self._get_message_stats(cutoff, source)
if not sessions:
@ -135,6 +136,15 @@ class InsightsEngine:
"models": [],
"platforms": [],
"tools": [],
"skills": {
"summary": {
"total_skill_loads": 0,
"total_skill_edits": 0,
"total_skill_actions": 0,
"distinct_skills_used": 0,
},
"top_skills": [],
},
"activity": {},
"top_sessions": [],
}
@ -144,6 +154,7 @@ class InsightsEngine:
models = self._compute_model_breakdown(sessions)
platforms = self._compute_platform_breakdown(sessions)
tools = self._compute_tool_breakdown(tool_usage)
skills = self._compute_skill_breakdown(skill_usage)
activity = self._compute_activity_patterns(sessions)
top_sessions = self._compute_top_sessions(sessions)
@ -156,6 +167,7 @@ class InsightsEngine:
"models": models,
"platforms": platforms,
"tools": tools,
"skills": skills,
"activity": activity,
"top_sessions": top_sessions,
}
@ -284,6 +296,82 @@ class InsightsEngine:
for name, count in tool_counts.most_common()
]
def _get_skill_usage(self, cutoff: float, source: str = None) -> List[Dict]:
"""Extract per-skill usage from assistant tool calls."""
skill_counts: Dict[str, Dict[str, Any]] = {}
if source:
cursor = self._conn.execute(
"""SELECT m.tool_calls, m.timestamp
FROM messages m
JOIN sessions s ON s.id = m.session_id
WHERE s.started_at >= ? AND s.source = ?
AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""",
(cutoff, source),
)
else:
cursor = self._conn.execute(
"""SELECT m.tool_calls, m.timestamp
FROM messages m
JOIN sessions s ON s.id = m.session_id
WHERE s.started_at >= ?
AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""",
(cutoff,),
)
for row in cursor.fetchall():
try:
calls = row["tool_calls"]
if isinstance(calls, str):
calls = json.loads(calls)
if not isinstance(calls, list):
continue
except (json.JSONDecodeError, TypeError):
continue
timestamp = row["timestamp"]
for call in calls:
if not isinstance(call, dict):
continue
func = call.get("function", {})
tool_name = func.get("name")
if tool_name not in {"skill_view", "skill_manage"}:
continue
args = func.get("arguments")
if isinstance(args, str):
try:
args = json.loads(args)
except (json.JSONDecodeError, TypeError):
continue
if not isinstance(args, dict):
continue
skill_name = args.get("name")
if not isinstance(skill_name, str) or not skill_name.strip():
continue
entry = skill_counts.setdefault(
skill_name,
{
"skill": skill_name,
"view_count": 0,
"manage_count": 0,
"last_used_at": None,
},
)
if tool_name == "skill_view":
entry["view_count"] += 1
else:
entry["manage_count"] += 1
if timestamp is not None and (
entry["last_used_at"] is None or timestamp > entry["last_used_at"]
):
entry["last_used_at"] = timestamp
return list(skill_counts.values())
def _get_message_stats(self, cutoff: float, source: str = None) -> Dict:
"""Get aggregate message statistics."""
if source:
@ -475,6 +563,46 @@ class InsightsEngine:
})
return result
def _compute_skill_breakdown(self, skill_usage: List[Dict]) -> Dict[str, Any]:
"""Process per-skill usage into summary + ranked list."""
total_skill_loads = sum(s["view_count"] for s in skill_usage) if skill_usage else 0
total_skill_edits = sum(s["manage_count"] for s in skill_usage) if skill_usage else 0
total_skill_actions = total_skill_loads + total_skill_edits
top_skills = []
for skill in skill_usage:
total_count = skill["view_count"] + skill["manage_count"]
percentage = (total_count / total_skill_actions * 100) if total_skill_actions else 0
top_skills.append({
"skill": skill["skill"],
"view_count": skill["view_count"],
"manage_count": skill["manage_count"],
"total_count": total_count,
"percentage": percentage,
"last_used_at": skill.get("last_used_at"),
})
top_skills.sort(
key=lambda s: (
s["total_count"],
s["view_count"],
s["manage_count"],
s["last_used_at"] or 0,
s["skill"],
),
reverse=True,
)
return {
"summary": {
"total_skill_loads": total_skill_loads,
"total_skill_edits": total_skill_edits,
"total_skill_actions": total_skill_actions,
"distinct_skills_used": len(skill_usage),
},
"top_skills": top_skills,
}
def _compute_activity_patterns(self, sessions: List[Dict]) -> Dict:
"""Analyze activity patterns by day of week and hour."""
day_counts = Counter() # 0=Monday ... 6=Sunday
@ -682,6 +810,28 @@ class InsightsEngine:
lines.append(f" ... and {len(report['tools']) - 15} more tools")
lines.append("")
# Skill usage
skills = report.get("skills", {})
top_skills = skills.get("top_skills", [])
if top_skills:
lines.append(" 🧠 Top Skills")
lines.append(" " + "" * 56)
lines.append(f" {'Skill':<28} {'Loads':>7} {'Edits':>7} {'Last used':>11}")
for skill in top_skills[:10]:
last_used = ""
if skill.get("last_used_at"):
last_used = datetime.fromtimestamp(skill["last_used_at"]).strftime("%b %d")
lines.append(
f" {skill['skill'][:28]:<28} {skill['view_count']:>7,} {skill['manage_count']:>7,} {last_used:>11}"
)
summary = skills.get("summary", {})
lines.append(
f" Distinct skills: {summary.get('distinct_skills_used', 0)} "
f"Loads: {summary.get('total_skill_loads', 0):,} "
f"Edits: {summary.get('total_skill_edits', 0):,}"
)
lines.append("")
# Activity patterns
act = report.get("activity", {})
if act.get("by_day"):
@ -774,6 +924,18 @@ class InsightsEngine:
lines.append(f" {t['tool']}{t['count']:,} calls ({t['percentage']:.1f}%)")
lines.append("")
skills = report.get("skills", {})
if skills.get("top_skills"):
lines.append("**🧠 Top Skills:**")
for skill in skills["top_skills"][:5]:
suffix = ""
if skill.get("last_used_at"):
suffix = f", last used {datetime.fromtimestamp(skill['last_used_at']).strftime('%b %d')}"
lines.append(
f" {skill['skill']}{skill['view_count']:,} loads, {skill['manage_count']:,} edits{suffix}"
)
lines.append("")
# Activity summary
act = report.get("activity", {})
if act.get("busiest_day") and act.get("busiest_hour"):

View file

@ -1977,6 +1977,8 @@ async def update_config_raw(body: RawConfigUpdate):
@app.get("/api/analytics/usage")
async def get_usage_analytics(days: int = 30):
from hermes_state import SessionDB
from agent.insights import InsightsEngine
db = SessionDB()
try:
cutoff = time.time() - (days * 86400)
@ -2016,8 +2018,24 @@ async def get_usage_analytics(days: int = 30):
FROM sessions WHERE started_at > ?
""", (cutoff,))
totals = dict(cur3.fetchone())
insights_report = InsightsEngine(db).generate(days=days)
skills = insights_report.get("skills", {
"summary": {
"total_skill_loads": 0,
"total_skill_edits": 0,
"total_skill_actions": 0,
"distinct_skills_used": 0,
},
"top_skills": [],
})
return {"daily": daily, "by_model": by_model, "totals": totals, "period_days": days}
return {
"daily": daily,
"by_model": by_model,
"totals": totals,
"period_days": days,
"skills": skills,
}
finally:
db.close()

View file

@ -51,6 +51,12 @@ def populated_db(db):
db.append_message("s1", role="assistant", content="I found the bug. Let me fix it.",
tool_calls=[{"function": {"name": "patch"}}])
db.append_message("s1", role="tool", content="patched successfully", tool_name="patch")
db.append_message(
"s1",
role="assistant",
content="Let me load the PR workflow skill.",
tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}}],
)
db.append_message("s1", role="user", content="Thanks!")
db.append_message("s1", role="assistant", content="You're welcome!")
@ -88,6 +94,12 @@ def populated_db(db):
db.append_message("s3", role="assistant", content="And search files",
tool_calls=[{"function": {"name": "search_files"}}])
db.append_message("s3", role="tool", content="found stuff", tool_name="search_files")
db.append_message(
"s3",
role="assistant",
content="Load the debugging skill.",
tool_calls=[{"function": {"name": "skill_view", "arguments": '{"name":"systematic-debugging"}'}}],
)
# Session 4: Discord, same model as s1, ended, 1 day ago
db.create_session(
@ -100,6 +112,15 @@ def populated_db(db):
db.update_token_counts("s4", input_tokens=10000, output_tokens=5000)
db.append_message("s4", role="user", content="Quick question")
db.append_message("s4", role="assistant", content="Sure, go ahead")
db.append_message(
"s4",
role="assistant",
content="Load and update GitHub skills.",
tool_calls=[
{"function": {"name": "skill_view", "arguments": '{"name":"github-pr-workflow"}'}},
{"function": {"name": "skill_manage", "arguments": '{"name":"github-code-review"}'}},
],
)
# Session 5: Old session, 45 days ago (should be excluded from 30-day window)
db.create_session(
@ -332,6 +353,35 @@ class TestInsightsPopulated:
total_pct = sum(t["percentage"] for t in tools)
assert total_pct == pytest.approx(100.0, abs=0.1)
def test_skill_breakdown(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
skills = report["skills"]
assert skills["summary"]["distinct_skills_used"] == 3
assert skills["summary"]["total_skill_loads"] == 3
assert skills["summary"]["total_skill_edits"] == 1
assert skills["summary"]["total_skill_actions"] == 4
top_skill = skills["top_skills"][0]
assert top_skill["skill"] == "github-pr-workflow"
assert top_skill["view_count"] == 2
assert top_skill["manage_count"] == 0
assert top_skill["total_count"] == 2
assert top_skill["last_used_at"] is not None
def test_skill_breakdown_respects_days_filter(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=3)
skills = report["skills"]
assert skills["summary"]["distinct_skills_used"] == 2
assert skills["summary"]["total_skill_loads"] == 2
assert skills["summary"]["total_skill_edits"] == 1
skill_names = [s["skill"] for s in skills["top_skills"]]
assert "systematic-debugging" not in skill_names
def test_activity_patterns(self, populated_db):
engine = InsightsEngine(populated_db)
report = engine.generate(days=30)
@ -401,6 +451,7 @@ class TestTerminalFormatting:
assert "Overview" in text
assert "Models Used" in text
assert "Top Tools" in text
assert "Top Skills" in text
assert "Activity Patterns" in text
assert "Notable Sessions" in text
@ -467,6 +518,7 @@ class TestGatewayFormatting:
text = engine.format_gateway(report)
assert "$" in text
assert "Top Skills" in text
assert "Est. cost" in text
def test_gateway_format_shows_models(self, populated_db):

View file

@ -101,14 +101,19 @@ class TestWebServerEndpoints:
"""Test the FastAPI REST endpoints using Starlette TestClient."""
@pytest.fixture(autouse=True)
def _setup_test_client(self):
"""Create a TestClient — import is deferred to avoid requiring fastapi."""
def _setup_test_client(self, monkeypatch, _isolate_hermes_home):
"""Create a TestClient and isolate the state DB under the test HERMES_HOME."""
try:
from starlette.testclient import TestClient
except ImportError:
pytest.skip("fastapi/starlette not installed")
import hermes_state
from hermes_constants import get_hermes_home
from hermes_cli.web_server import app, _SESSION_TOKEN
monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db")
self.client = TestClient(app)
self.client.headers["Authorization"] = f"Bearer {_SESSION_TOKEN}"
@ -511,12 +516,18 @@ class TestNewEndpoints:
"""Tests for session detail, logs, cron, skills, tools, raw config, analytics."""
@pytest.fixture(autouse=True)
def _setup(self):
def _setup(self, monkeypatch, _isolate_hermes_home):
try:
from starlette.testclient import TestClient
except ImportError:
pytest.skip("fastapi/starlette not installed")
import hermes_state
from hermes_constants import get_hermes_home
from hermes_cli.web_server import app, _SESSION_TOKEN
monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db")
self.client = TestClient(app)
self.client.headers["Authorization"] = f"Bearer {_SESSION_TOKEN}"
@ -692,8 +703,74 @@ class TestNewEndpoints:
assert "daily" in data
assert "by_model" in data
assert "totals" in data
assert "skills" in data
assert isinstance(data["daily"], list)
assert "total_sessions" in data["totals"]
assert data["skills"] == {
"summary": {
"total_skill_loads": 0,
"total_skill_edits": 0,
"total_skill_actions": 0,
"distinct_skills_used": 0,
},
"top_skills": [],
}
def test_analytics_usage_includes_skill_breakdown(self):
from hermes_state import SessionDB
db = SessionDB()
try:
db.create_session(
session_id="skills-analytics-test",
source="cli",
model="anthropic/claude-sonnet-4",
)
db.update_token_counts(
"skills-analytics-test",
input_tokens=120,
output_tokens=45,
)
db.append_message(
"skills-analytics-test",
role="assistant",
content="Loading and updating skills.",
tool_calls=[
{
"function": {
"name": "skill_view",
"arguments": '{"name":"github-pr-workflow"}',
}
},
{
"function": {
"name": "skill_manage",
"arguments": '{"name":"github-code-review"}',
}
},
],
)
finally:
db.close()
resp = self.client.get("/api/analytics/usage?days=7")
assert resp.status_code == 200
data = resp.json()
assert data["skills"]["summary"] == {
"total_skill_loads": 1,
"total_skill_edits": 1,
"total_skill_actions": 2,
"distinct_skills_used": 2,
}
assert len(data["skills"]["top_skills"]) == 2
top_skill = data["skills"]["top_skills"][0]
assert top_skill["skill"] == "github-pr-workflow"
assert top_skill["view_count"] == 1
assert top_skill["manage_count"] == 0
assert top_skill["total_count"] == 1
assert top_skill["last_used_at"] is not None
def test_session_token_endpoint_removed(self):
"""GET /api/auth/session-token no longer exists."""

View file

@ -115,6 +115,11 @@ export const en: Translations = {
dailyTokenUsage: "Daily Token Usage",
dailyBreakdown: "Daily Breakdown",
perModelBreakdown: "Per-Model Breakdown",
topSkills: "Top Skills",
skill: "Skill",
loads: "Agent Loaded",
edits: "Agent Managed",
lastUsed: "Last Used",
input: "Input",
output: "Output",
total: "Total",

View file

@ -120,6 +120,11 @@ export interface Translations {
dailyTokenUsage: string;
dailyBreakdown: string;
perModelBreakdown: string;
topSkills: string;
skill: string;
loads: string;
edits: string;
lastUsed: string;
input: string;
output: string;
total: string;

View file

@ -115,6 +115,11 @@ export const zh: Translations = {
dailyTokenUsage: "每日 Token 用量",
dailyBreakdown: "每日明细",
perModelBreakdown: "模型用量明细",
topSkills: "常用技能",
skill: "技能",
loads: "代理加载",
edits: "代理管理",
lastUsed: "最近使用",
input: "输入",
output: "输出",
total: "总计",

View file

@ -283,6 +283,22 @@ export interface AnalyticsModelEntry {
sessions: number;
}
export interface AnalyticsSkillEntry {
skill: string;
view_count: number;
manage_count: number;
total_count: number;
percentage: number;
last_used_at: number | null;
}
export interface AnalyticsSkillsSummary {
total_skill_loads: number;
total_skill_edits: number;
total_skill_actions: number;
distinct_skills_used: number;
}
export interface AnalyticsResponse {
daily: AnalyticsDailyEntry[];
by_model: AnalyticsModelEntry[];
@ -295,6 +311,10 @@ export interface AnalyticsResponse {
total_actual_cost: number;
total_sessions: number;
};
skills: {
summary: AnalyticsSkillsSummary;
top_skills: AnalyticsSkillEntry[];
};
}
export interface CronJob {

View file

@ -1,12 +1,14 @@
import { useEffect, useState, useCallback } from "react";
import {
BarChart3,
Brain,
Cpu,
Hash,
TrendingUp,
} from "lucide-react";
import { api } from "@/lib/api";
import type { AnalyticsResponse, AnalyticsDailyEntry, AnalyticsModelEntry } from "@/lib/api";
import type { AnalyticsResponse, AnalyticsDailyEntry, AnalyticsModelEntry, AnalyticsSkillEntry } from "@/lib/api";
import { timeAgo } from "@/lib/utils";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Button } from "@/components/ui/button";
import { useI18n } from "@/i18n";
@ -227,6 +229,52 @@ function ModelTable({ models }: { models: AnalyticsModelEntry[] }) {
);
}
function SkillTable({ skills }: { skills: AnalyticsSkillEntry[] }) {
const { t } = useI18n();
if (skills.length === 0) return null;
return (
<Card>
<CardHeader>
<div className="flex items-center gap-2">
<Brain className="h-5 w-5 text-muted-foreground" />
<CardTitle className="text-base">{t.analytics.topSkills}</CardTitle>
</div>
</CardHeader>
<CardContent>
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead>
<tr className="border-b border-border text-muted-foreground text-xs">
<th className="text-left py-2 pr-4 font-medium">{t.analytics.skill}</th>
<th className="text-right py-2 px-4 font-medium">{t.analytics.loads}</th>
<th className="text-right py-2 px-4 font-medium">{t.analytics.edits}</th>
<th className="text-right py-2 px-4 font-medium">{t.analytics.total}</th>
<th className="text-right py-2 pl-4 font-medium">{t.analytics.lastUsed}</th>
</tr>
</thead>
<tbody>
{skills.map((skill) => (
<tr key={skill.skill} className="border-b border-border/50 hover:bg-secondary/20 transition-colors">
<td className="py-2 pr-4">
<span className="font-mono-ui text-xs">{skill.skill}</span>
</td>
<td className="text-right py-2 px-4 text-muted-foreground">{skill.view_count}</td>
<td className="text-right py-2 px-4 text-muted-foreground">{skill.manage_count}</td>
<td className="text-right py-2 px-4">{skill.total_count}</td>
<td className="text-right py-2 pl-4 text-muted-foreground">
{skill.last_used_at ? timeAgo(skill.last_used_at) : "—"}
</td>
</tr>
))}
</tbody>
</table>
</div>
</CardContent>
</Card>
);
}
export default function AnalyticsPage() {
const [days, setDays] = useState(30);
const [data, setData] = useState<AnalyticsResponse | null>(null);
@ -310,10 +358,11 @@ export default function AnalyticsPage() {
{/* Tables */}
<DailyTable daily={data.daily} />
<ModelTable models={data.by_model} />
<SkillTable skills={data.skills.top_skills} />
</>
)}
{data && data.daily.length === 0 && data.by_model.length === 0 && (
{data && data.daily.length === 0 && data.by_model.length === 0 && data.skills.top_skills.length === 0 && (
<Card>
<CardContent className="py-12">
<div className="flex flex-col items-center text-muted-foreground">