feat(dashboard): track real API call count per session

Adds schema v7 'api_call_count' column. run_agent.py increments it by 1
per LLM API call, web_server analytics SQL aggregates it, frontend uses
the real counter instead of summing sessions.

The 'API Calls' card on the analytics dashboard previously displayed
COUNT(*) from the sessions table — the number of conversations, not
LLM requests. Each session makes 10-90 API calls through the tool loop,
so the reported number was ~30x lower than real.

Salvaged from PR #10140 (@kshitijk4poor). The cache-token accuracy
portions of the original PR were deferred — per-provider analytics is
the better path there, since cache_write_tokens and actual_cost_usd
are only reliably available from a subset of providers (Anthropic
native, Codex Responses, OpenRouter with usage.include).

Tests:
- schema_version v7 assertion
- migration v2 -> v7 adds api_call_count column with default 0
- update_token_counts increments api_call_count by provided delta
- absolute=True sets api_call_count directly
- /api/analytics/usage exposes total_api_calls in totals
This commit is contained in:
kshitijk4poor 2026-04-22 05:29:21 -07:00 committed by Teknium
parent be11a75eae
commit 5fb143169b
7 changed files with 61 additions and 10 deletions

View file

@ -2189,7 +2189,8 @@ async def get_usage_analytics(days: int = 30):
SUM(reasoning_tokens) as reasoning_tokens, SUM(reasoning_tokens) as reasoning_tokens,
COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost, COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost,
COALESCE(SUM(actual_cost_usd), 0) as actual_cost, COALESCE(SUM(actual_cost_usd), 0) as actual_cost,
COUNT(*) as sessions COUNT(*) as sessions,
SUM(COALESCE(api_call_count, 0)) as api_calls
FROM sessions WHERE started_at > ? FROM sessions WHERE started_at > ?
GROUP BY day ORDER BY day GROUP BY day ORDER BY day
""", (cutoff,)) """, (cutoff,))
@ -2200,7 +2201,8 @@ async def get_usage_analytics(days: int = 30):
SUM(input_tokens) as input_tokens, SUM(input_tokens) as input_tokens,
SUM(output_tokens) as output_tokens, SUM(output_tokens) as output_tokens,
COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost, COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost,
COUNT(*) as sessions COUNT(*) as sessions,
SUM(COALESCE(api_call_count, 0)) as api_calls
FROM sessions WHERE started_at > ? AND model IS NOT NULL FROM sessions WHERE started_at > ? AND model IS NOT NULL
GROUP BY model ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC GROUP BY model ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC
""", (cutoff,)) """, (cutoff,))
@ -2213,7 +2215,8 @@ async def get_usage_analytics(days: int = 30):
SUM(reasoning_tokens) as total_reasoning, SUM(reasoning_tokens) as total_reasoning,
COALESCE(SUM(estimated_cost_usd), 0) as total_estimated_cost, COALESCE(SUM(estimated_cost_usd), 0) as total_estimated_cost,
COALESCE(SUM(actual_cost_usd), 0) as total_actual_cost, COALESCE(SUM(actual_cost_usd), 0) as total_actual_cost,
COUNT(*) as total_sessions COUNT(*) as total_sessions,
SUM(COALESCE(api_call_count, 0)) as total_api_calls
FROM sessions WHERE started_at > ? FROM sessions WHERE started_at > ?
""", (cutoff,)) """, (cutoff,))
totals = dict(cur3.fetchone()) totals = dict(cur3.fetchone())

View file

@ -31,7 +31,7 @@ T = TypeVar("T")
DEFAULT_DB_PATH = get_hermes_home() / "state.db" DEFAULT_DB_PATH = get_hermes_home() / "state.db"
SCHEMA_VERSION = 7 SCHEMA_VERSION = 8
SCHEMA_SQL = """ SCHEMA_SQL = """
CREATE TABLE IF NOT EXISTS schema_version ( CREATE TABLE IF NOT EXISTS schema_version (
@ -65,6 +65,7 @@ CREATE TABLE IF NOT EXISTS sessions (
cost_source TEXT, cost_source TEXT,
pricing_version TEXT, pricing_version TEXT,
title TEXT, title TEXT,
api_call_count INTEGER DEFAULT 0,
FOREIGN KEY (parent_session_id) REFERENCES sessions(id) FOREIGN KEY (parent_session_id) REFERENCES sessions(id)
); );
@ -344,6 +345,17 @@ class SessionDB:
except sqlite3.OperationalError: except sqlite3.OperationalError:
pass # Column already exists pass # Column already exists
cursor.execute("UPDATE schema_version SET version = 7") cursor.execute("UPDATE schema_version SET version = 7")
if current_version < 8:
# v8: add api_call_count column to sessions — tracks the number
# of individual LLM API calls made within a session (as opposed
# to the session count itself).
try:
cursor.execute(
'ALTER TABLE sessions ADD COLUMN "api_call_count" INTEGER DEFAULT 0'
)
except sqlite3.OperationalError:
pass # Column already exists
cursor.execute("UPDATE schema_version SET version = 8")
# Unique title index — always ensure it exists (safe to run after migrations # Unique title index — always ensure it exists (safe to run after migrations
# since the title column is guaranteed to exist at this point) # since the title column is guaranteed to exist at this point)
@ -450,6 +462,7 @@ class SessionDB:
billing_provider: Optional[str] = None, billing_provider: Optional[str] = None,
billing_base_url: Optional[str] = None, billing_base_url: Optional[str] = None,
billing_mode: Optional[str] = None, billing_mode: Optional[str] = None,
api_call_count: int = 0,
absolute: bool = False, absolute: bool = False,
) -> None: ) -> None:
"""Update token counters and backfill model if not already set. """Update token counters and backfill model if not already set.
@ -479,7 +492,8 @@ class SessionDB:
billing_provider = COALESCE(billing_provider, ?), billing_provider = COALESCE(billing_provider, ?),
billing_base_url = COALESCE(billing_base_url, ?), billing_base_url = COALESCE(billing_base_url, ?),
billing_mode = COALESCE(billing_mode, ?), billing_mode = COALESCE(billing_mode, ?),
model = COALESCE(model, ?) model = COALESCE(model, ?),
api_call_count = ?
WHERE id = ?""" WHERE id = ?"""
else: else:
sql = """UPDATE sessions SET sql = """UPDATE sessions SET
@ -499,7 +513,8 @@ class SessionDB:
billing_provider = COALESCE(billing_provider, ?), billing_provider = COALESCE(billing_provider, ?),
billing_base_url = COALESCE(billing_base_url, ?), billing_base_url = COALESCE(billing_base_url, ?),
billing_mode = COALESCE(billing_mode, ?), billing_mode = COALESCE(billing_mode, ?),
model = COALESCE(model, ?) model = COALESCE(model, ?),
api_call_count = COALESCE(api_call_count, 0) + ?
WHERE id = ?""" WHERE id = ?"""
params = ( params = (
input_tokens, input_tokens,
@ -517,6 +532,7 @@ class SessionDB:
billing_base_url, billing_base_url,
billing_mode, billing_mode,
model, model,
api_call_count,
session_id, session_id,
) )
def _do(conn): def _do(conn):

View file

@ -9767,6 +9767,7 @@ class AIAgent:
billing_mode="subscription_included" billing_mode="subscription_included"
if cost_result.status == "included" else None, if cost_result.status == "included" else None,
model=self.model, model=self.model,
api_call_count=1,
) )
except Exception: except Exception:
pass # never block the agent loop pass # never block the agent loop

View file

@ -706,6 +706,7 @@ class TestNewEndpoints:
assert "skills" in data assert "skills" in data
assert isinstance(data["daily"], list) assert isinstance(data["daily"], list)
assert "total_sessions" in data["totals"] assert "total_sessions" in data["totals"]
assert "total_api_calls" in data["totals"]
assert data["skills"] == { assert data["skills"] == {
"summary": { "summary": {
"total_skill_loads": 0, "total_skill_loads": 0,

View file

@ -93,6 +93,27 @@ class TestSessionLifecycle:
assert session["input_tokens"] == 300 assert session["input_tokens"] == 300
assert session["output_tokens"] == 150 assert session["output_tokens"] == 150
def test_update_token_counts_tracks_api_call_count(self, db):
"""api_call_count increments with each update_token_counts call."""
db.create_session(session_id="s1", source="cli")
db.update_token_counts("s1", input_tokens=100, output_tokens=50, api_call_count=1)
db.update_token_counts("s1", input_tokens=100, output_tokens=50, api_call_count=1)
db.update_token_counts("s1", input_tokens=100, output_tokens=50, api_call_count=1)
session = db.get_session("s1")
assert session["api_call_count"] == 3
def test_update_token_counts_api_call_count_absolute(self, db):
"""absolute mode sets api_call_count directly."""
db.create_session(session_id="s1", source="cli")
db.update_token_counts("s1", input_tokens=100, output_tokens=50, api_call_count=1)
db.update_token_counts("s1", input_tokens=300, output_tokens=150,
api_call_count=5, absolute=True)
session = db.get_session("s1")
assert session["api_call_count"] == 5
assert session["input_tokens"] == 300
def test_update_token_counts_backfills_model_when_null(self, db): def test_update_token_counts_backfills_model_when_null(self, db):
db.create_session(session_id="s1", source="telegram") db.create_session(session_id="s1", source="telegram")
db.update_token_counts("s1", input_tokens=10, output_tokens=5, model="openai/gpt-5.4") db.update_token_counts("s1", input_tokens=10, output_tokens=5, model="openai/gpt-5.4")
@ -1152,7 +1173,7 @@ class TestSchemaInit:
def test_schema_version(self, db): def test_schema_version(self, db):
cursor = db._conn.execute("SELECT version FROM schema_version") cursor = db._conn.execute("SELECT version FROM schema_version")
version = cursor.fetchone()[0] version = cursor.fetchone()[0]
assert version == 7 assert version == 8
def test_title_column_exists(self, db): def test_title_column_exists(self, db):
"""Verify the title column was created in the sessions table.""" """Verify the title column was created in the sessions table."""
@ -1208,18 +1229,24 @@ class TestSchemaInit:
conn.commit() conn.commit()
conn.close() conn.close()
# Open with SessionDB — should migrate to v7 # Open with SessionDB — should migrate to v8
migrated_db = SessionDB(db_path=db_path) migrated_db = SessionDB(db_path=db_path)
# Verify migration # Verify migration
cursor = migrated_db._conn.execute("SELECT version FROM schema_version") cursor = migrated_db._conn.execute("SELECT version FROM schema_version")
assert cursor.fetchone()[0] == 7 assert cursor.fetchone()[0] == 8
# Verify title column exists and is NULL for existing sessions # Verify title column exists and is NULL for existing sessions
session = migrated_db.get_session("existing") session = migrated_db.get_session("existing")
assert session is not None assert session is not None
assert session["title"] is None assert session["title"] is None
# Verify api_call_count column was added with default 0
cursor = migrated_db._conn.execute(
"SELECT api_call_count FROM sessions WHERE id = 'existing'"
)
assert cursor.fetchone()[0] == 0
# Verify we can set title on migrated session # Verify we can set title on migrated session
assert migrated_db.set_session_title("existing", "Migrated Title") is True assert migrated_db.set_session_title("existing", "Migrated Title") is True
session = migrated_db.get_session("existing") session = migrated_db.get_session("existing")

View file

@ -314,6 +314,7 @@ export interface AnalyticsDailyEntry {
estimated_cost: number; estimated_cost: number;
actual_cost: number; actual_cost: number;
sessions: number; sessions: number;
api_calls: number;
} }
export interface AnalyticsModelEntry { export interface AnalyticsModelEntry {
@ -322,6 +323,7 @@ export interface AnalyticsModelEntry {
output_tokens: number; output_tokens: number;
estimated_cost: number; estimated_cost: number;
sessions: number; sessions: number;
api_calls: number;
} }
export interface AnalyticsSkillEntry { export interface AnalyticsSkillEntry {
@ -351,6 +353,7 @@ export interface AnalyticsResponse {
total_estimated_cost: number; total_estimated_cost: number;
total_actual_cost: number; total_actual_cost: number;
total_sessions: number; total_sessions: number;
total_api_calls: number;
}; };
skills: { skills: {
summary: AnalyticsSkillsSummary; summary: AnalyticsSkillsSummary;

View file

@ -347,7 +347,7 @@ export default function AnalyticsPage() {
<SummaryCard <SummaryCard
icon={TrendingUp} icon={TrendingUp}
label={t.analytics.apiCalls} label={t.analytics.apiCalls}
value={String(data.daily.reduce((sum, d) => sum + d.sessions, 0))} value={String(data.totals.total_api_calls ?? data.daily.reduce((sum, d) => sum + d.sessions, 0))}
sub={t.analytics.acrossModels.replace("{count}", String(data.by_model.length))} sub={t.analytics.acrossModels.replace("{count}", String(data.by_model.length))}
/> />
</div> </div>