mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-25 00:51:20 +00:00
feat(dashboard): track real API call count per session
Adds schema v7 'api_call_count' column. run_agent.py increments it by 1 per LLM API call, web_server analytics SQL aggregates it, frontend uses the real counter instead of summing sessions. The 'API Calls' card on the analytics dashboard previously displayed COUNT(*) from the sessions table — the number of conversations, not LLM requests. Each session makes 10-90 API calls through the tool loop, so the reported number was ~30x lower than real. Salvaged from PR #10140 (@kshitijk4poor). The cache-token accuracy portions of the original PR were deferred — per-provider analytics is the better path there, since cache_write_tokens and actual_cost_usd are only reliably available from a subset of providers (Anthropic native, Codex Responses, OpenRouter with usage.include). Tests: - schema_version v7 assertion - migration v2 -> v7 adds api_call_count column with default 0 - update_token_counts increments api_call_count by provided delta - absolute=True sets api_call_count directly - /api/analytics/usage exposes total_api_calls in totals
This commit is contained in:
parent
be11a75eae
commit
5fb143169b
7 changed files with 61 additions and 10 deletions
|
|
@ -2189,7 +2189,8 @@ async def get_usage_analytics(days: int = 30):
|
||||||
SUM(reasoning_tokens) as reasoning_tokens,
|
SUM(reasoning_tokens) as reasoning_tokens,
|
||||||
COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost,
|
COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost,
|
||||||
COALESCE(SUM(actual_cost_usd), 0) as actual_cost,
|
COALESCE(SUM(actual_cost_usd), 0) as actual_cost,
|
||||||
COUNT(*) as sessions
|
COUNT(*) as sessions,
|
||||||
|
SUM(COALESCE(api_call_count, 0)) as api_calls
|
||||||
FROM sessions WHERE started_at > ?
|
FROM sessions WHERE started_at > ?
|
||||||
GROUP BY day ORDER BY day
|
GROUP BY day ORDER BY day
|
||||||
""", (cutoff,))
|
""", (cutoff,))
|
||||||
|
|
@ -2200,7 +2201,8 @@ async def get_usage_analytics(days: int = 30):
|
||||||
SUM(input_tokens) as input_tokens,
|
SUM(input_tokens) as input_tokens,
|
||||||
SUM(output_tokens) as output_tokens,
|
SUM(output_tokens) as output_tokens,
|
||||||
COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost,
|
COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost,
|
||||||
COUNT(*) as sessions
|
COUNT(*) as sessions,
|
||||||
|
SUM(COALESCE(api_call_count, 0)) as api_calls
|
||||||
FROM sessions WHERE started_at > ? AND model IS NOT NULL
|
FROM sessions WHERE started_at > ? AND model IS NOT NULL
|
||||||
GROUP BY model ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC
|
GROUP BY model ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC
|
||||||
""", (cutoff,))
|
""", (cutoff,))
|
||||||
|
|
@ -2213,7 +2215,8 @@ async def get_usage_analytics(days: int = 30):
|
||||||
SUM(reasoning_tokens) as total_reasoning,
|
SUM(reasoning_tokens) as total_reasoning,
|
||||||
COALESCE(SUM(estimated_cost_usd), 0) as total_estimated_cost,
|
COALESCE(SUM(estimated_cost_usd), 0) as total_estimated_cost,
|
||||||
COALESCE(SUM(actual_cost_usd), 0) as total_actual_cost,
|
COALESCE(SUM(actual_cost_usd), 0) as total_actual_cost,
|
||||||
COUNT(*) as total_sessions
|
COUNT(*) as total_sessions,
|
||||||
|
SUM(COALESCE(api_call_count, 0)) as total_api_calls
|
||||||
FROM sessions WHERE started_at > ?
|
FROM sessions WHERE started_at > ?
|
||||||
""", (cutoff,))
|
""", (cutoff,))
|
||||||
totals = dict(cur3.fetchone())
|
totals = dict(cur3.fetchone())
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@ T = TypeVar("T")
|
||||||
|
|
||||||
DEFAULT_DB_PATH = get_hermes_home() / "state.db"
|
DEFAULT_DB_PATH = get_hermes_home() / "state.db"
|
||||||
|
|
||||||
SCHEMA_VERSION = 7
|
SCHEMA_VERSION = 8
|
||||||
|
|
||||||
SCHEMA_SQL = """
|
SCHEMA_SQL = """
|
||||||
CREATE TABLE IF NOT EXISTS schema_version (
|
CREATE TABLE IF NOT EXISTS schema_version (
|
||||||
|
|
@ -65,6 +65,7 @@ CREATE TABLE IF NOT EXISTS sessions (
|
||||||
cost_source TEXT,
|
cost_source TEXT,
|
||||||
pricing_version TEXT,
|
pricing_version TEXT,
|
||||||
title TEXT,
|
title TEXT,
|
||||||
|
api_call_count INTEGER DEFAULT 0,
|
||||||
FOREIGN KEY (parent_session_id) REFERENCES sessions(id)
|
FOREIGN KEY (parent_session_id) REFERENCES sessions(id)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -344,6 +345,17 @@ class SessionDB:
|
||||||
except sqlite3.OperationalError:
|
except sqlite3.OperationalError:
|
||||||
pass # Column already exists
|
pass # Column already exists
|
||||||
cursor.execute("UPDATE schema_version SET version = 7")
|
cursor.execute("UPDATE schema_version SET version = 7")
|
||||||
|
if current_version < 8:
|
||||||
|
# v8: add api_call_count column to sessions — tracks the number
|
||||||
|
# of individual LLM API calls made within a session (as opposed
|
||||||
|
# to the session count itself).
|
||||||
|
try:
|
||||||
|
cursor.execute(
|
||||||
|
'ALTER TABLE sessions ADD COLUMN "api_call_count" INTEGER DEFAULT 0'
|
||||||
|
)
|
||||||
|
except sqlite3.OperationalError:
|
||||||
|
pass # Column already exists
|
||||||
|
cursor.execute("UPDATE schema_version SET version = 8")
|
||||||
|
|
||||||
# Unique title index — always ensure it exists (safe to run after migrations
|
# Unique title index — always ensure it exists (safe to run after migrations
|
||||||
# since the title column is guaranteed to exist at this point)
|
# since the title column is guaranteed to exist at this point)
|
||||||
|
|
@ -450,6 +462,7 @@ class SessionDB:
|
||||||
billing_provider: Optional[str] = None,
|
billing_provider: Optional[str] = None,
|
||||||
billing_base_url: Optional[str] = None,
|
billing_base_url: Optional[str] = None,
|
||||||
billing_mode: Optional[str] = None,
|
billing_mode: Optional[str] = None,
|
||||||
|
api_call_count: int = 0,
|
||||||
absolute: bool = False,
|
absolute: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Update token counters and backfill model if not already set.
|
"""Update token counters and backfill model if not already set.
|
||||||
|
|
@ -479,7 +492,8 @@ class SessionDB:
|
||||||
billing_provider = COALESCE(billing_provider, ?),
|
billing_provider = COALESCE(billing_provider, ?),
|
||||||
billing_base_url = COALESCE(billing_base_url, ?),
|
billing_base_url = COALESCE(billing_base_url, ?),
|
||||||
billing_mode = COALESCE(billing_mode, ?),
|
billing_mode = COALESCE(billing_mode, ?),
|
||||||
model = COALESCE(model, ?)
|
model = COALESCE(model, ?),
|
||||||
|
api_call_count = ?
|
||||||
WHERE id = ?"""
|
WHERE id = ?"""
|
||||||
else:
|
else:
|
||||||
sql = """UPDATE sessions SET
|
sql = """UPDATE sessions SET
|
||||||
|
|
@ -499,7 +513,8 @@ class SessionDB:
|
||||||
billing_provider = COALESCE(billing_provider, ?),
|
billing_provider = COALESCE(billing_provider, ?),
|
||||||
billing_base_url = COALESCE(billing_base_url, ?),
|
billing_base_url = COALESCE(billing_base_url, ?),
|
||||||
billing_mode = COALESCE(billing_mode, ?),
|
billing_mode = COALESCE(billing_mode, ?),
|
||||||
model = COALESCE(model, ?)
|
model = COALESCE(model, ?),
|
||||||
|
api_call_count = COALESCE(api_call_count, 0) + ?
|
||||||
WHERE id = ?"""
|
WHERE id = ?"""
|
||||||
params = (
|
params = (
|
||||||
input_tokens,
|
input_tokens,
|
||||||
|
|
@ -517,6 +532,7 @@ class SessionDB:
|
||||||
billing_base_url,
|
billing_base_url,
|
||||||
billing_mode,
|
billing_mode,
|
||||||
model,
|
model,
|
||||||
|
api_call_count,
|
||||||
session_id,
|
session_id,
|
||||||
)
|
)
|
||||||
def _do(conn):
|
def _do(conn):
|
||||||
|
|
|
||||||
|
|
@ -9767,6 +9767,7 @@ class AIAgent:
|
||||||
billing_mode="subscription_included"
|
billing_mode="subscription_included"
|
||||||
if cost_result.status == "included" else None,
|
if cost_result.status == "included" else None,
|
||||||
model=self.model,
|
model=self.model,
|
||||||
|
api_call_count=1,
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass # never block the agent loop
|
pass # never block the agent loop
|
||||||
|
|
|
||||||
|
|
@ -706,6 +706,7 @@ class TestNewEndpoints:
|
||||||
assert "skills" in data
|
assert "skills" in data
|
||||||
assert isinstance(data["daily"], list)
|
assert isinstance(data["daily"], list)
|
||||||
assert "total_sessions" in data["totals"]
|
assert "total_sessions" in data["totals"]
|
||||||
|
assert "total_api_calls" in data["totals"]
|
||||||
assert data["skills"] == {
|
assert data["skills"] == {
|
||||||
"summary": {
|
"summary": {
|
||||||
"total_skill_loads": 0,
|
"total_skill_loads": 0,
|
||||||
|
|
|
||||||
|
|
@ -93,6 +93,27 @@ class TestSessionLifecycle:
|
||||||
assert session["input_tokens"] == 300
|
assert session["input_tokens"] == 300
|
||||||
assert session["output_tokens"] == 150
|
assert session["output_tokens"] == 150
|
||||||
|
|
||||||
|
def test_update_token_counts_tracks_api_call_count(self, db):
|
||||||
|
"""api_call_count increments with each update_token_counts call."""
|
||||||
|
db.create_session(session_id="s1", source="cli")
|
||||||
|
db.update_token_counts("s1", input_tokens=100, output_tokens=50, api_call_count=1)
|
||||||
|
db.update_token_counts("s1", input_tokens=100, output_tokens=50, api_call_count=1)
|
||||||
|
db.update_token_counts("s1", input_tokens=100, output_tokens=50, api_call_count=1)
|
||||||
|
|
||||||
|
session = db.get_session("s1")
|
||||||
|
assert session["api_call_count"] == 3
|
||||||
|
|
||||||
|
def test_update_token_counts_api_call_count_absolute(self, db):
|
||||||
|
"""absolute mode sets api_call_count directly."""
|
||||||
|
db.create_session(session_id="s1", source="cli")
|
||||||
|
db.update_token_counts("s1", input_tokens=100, output_tokens=50, api_call_count=1)
|
||||||
|
db.update_token_counts("s1", input_tokens=300, output_tokens=150,
|
||||||
|
api_call_count=5, absolute=True)
|
||||||
|
|
||||||
|
session = db.get_session("s1")
|
||||||
|
assert session["api_call_count"] == 5
|
||||||
|
assert session["input_tokens"] == 300
|
||||||
|
|
||||||
def test_update_token_counts_backfills_model_when_null(self, db):
|
def test_update_token_counts_backfills_model_when_null(self, db):
|
||||||
db.create_session(session_id="s1", source="telegram")
|
db.create_session(session_id="s1", source="telegram")
|
||||||
db.update_token_counts("s1", input_tokens=10, output_tokens=5, model="openai/gpt-5.4")
|
db.update_token_counts("s1", input_tokens=10, output_tokens=5, model="openai/gpt-5.4")
|
||||||
|
|
@ -1152,7 +1173,7 @@ class TestSchemaInit:
|
||||||
def test_schema_version(self, db):
|
def test_schema_version(self, db):
|
||||||
cursor = db._conn.execute("SELECT version FROM schema_version")
|
cursor = db._conn.execute("SELECT version FROM schema_version")
|
||||||
version = cursor.fetchone()[0]
|
version = cursor.fetchone()[0]
|
||||||
assert version == 7
|
assert version == 8
|
||||||
|
|
||||||
def test_title_column_exists(self, db):
|
def test_title_column_exists(self, db):
|
||||||
"""Verify the title column was created in the sessions table."""
|
"""Verify the title column was created in the sessions table."""
|
||||||
|
|
@ -1208,18 +1229,24 @@ class TestSchemaInit:
|
||||||
conn.commit()
|
conn.commit()
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
# Open with SessionDB — should migrate to v7
|
# Open with SessionDB — should migrate to v8
|
||||||
migrated_db = SessionDB(db_path=db_path)
|
migrated_db = SessionDB(db_path=db_path)
|
||||||
|
|
||||||
# Verify migration
|
# Verify migration
|
||||||
cursor = migrated_db._conn.execute("SELECT version FROM schema_version")
|
cursor = migrated_db._conn.execute("SELECT version FROM schema_version")
|
||||||
assert cursor.fetchone()[0] == 7
|
assert cursor.fetchone()[0] == 8
|
||||||
|
|
||||||
# Verify title column exists and is NULL for existing sessions
|
# Verify title column exists and is NULL for existing sessions
|
||||||
session = migrated_db.get_session("existing")
|
session = migrated_db.get_session("existing")
|
||||||
assert session is not None
|
assert session is not None
|
||||||
assert session["title"] is None
|
assert session["title"] is None
|
||||||
|
|
||||||
|
# Verify api_call_count column was added with default 0
|
||||||
|
cursor = migrated_db._conn.execute(
|
||||||
|
"SELECT api_call_count FROM sessions WHERE id = 'existing'"
|
||||||
|
)
|
||||||
|
assert cursor.fetchone()[0] == 0
|
||||||
|
|
||||||
# Verify we can set title on migrated session
|
# Verify we can set title on migrated session
|
||||||
assert migrated_db.set_session_title("existing", "Migrated Title") is True
|
assert migrated_db.set_session_title("existing", "Migrated Title") is True
|
||||||
session = migrated_db.get_session("existing")
|
session = migrated_db.get_session("existing")
|
||||||
|
|
|
||||||
|
|
@ -314,6 +314,7 @@ export interface AnalyticsDailyEntry {
|
||||||
estimated_cost: number;
|
estimated_cost: number;
|
||||||
actual_cost: number;
|
actual_cost: number;
|
||||||
sessions: number;
|
sessions: number;
|
||||||
|
api_calls: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AnalyticsModelEntry {
|
export interface AnalyticsModelEntry {
|
||||||
|
|
@ -322,6 +323,7 @@ export interface AnalyticsModelEntry {
|
||||||
output_tokens: number;
|
output_tokens: number;
|
||||||
estimated_cost: number;
|
estimated_cost: number;
|
||||||
sessions: number;
|
sessions: number;
|
||||||
|
api_calls: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AnalyticsSkillEntry {
|
export interface AnalyticsSkillEntry {
|
||||||
|
|
@ -351,6 +353,7 @@ export interface AnalyticsResponse {
|
||||||
total_estimated_cost: number;
|
total_estimated_cost: number;
|
||||||
total_actual_cost: number;
|
total_actual_cost: number;
|
||||||
total_sessions: number;
|
total_sessions: number;
|
||||||
|
total_api_calls: number;
|
||||||
};
|
};
|
||||||
skills: {
|
skills: {
|
||||||
summary: AnalyticsSkillsSummary;
|
summary: AnalyticsSkillsSummary;
|
||||||
|
|
|
||||||
|
|
@ -347,7 +347,7 @@ export default function AnalyticsPage() {
|
||||||
<SummaryCard
|
<SummaryCard
|
||||||
icon={TrendingUp}
|
icon={TrendingUp}
|
||||||
label={t.analytics.apiCalls}
|
label={t.analytics.apiCalls}
|
||||||
value={String(data.daily.reduce((sum, d) => sum + d.sessions, 0))}
|
value={String(data.totals.total_api_calls ?? data.daily.reduce((sum, d) => sum + d.sessions, 0))}
|
||||||
sub={t.analytics.acrossModels.replace("{count}", String(data.by_model.length))}
|
sub={t.analytics.acrossModels.replace("{count}", String(data.by_model.length))}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue