From 7dfd5077ceef4d5a6f7953c050bad1a75e86e215 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:10:51 -0700 Subject: [PATCH] feat(oneshot): add --usage-file JSON usage report to hermes -z (#59615) * feat(oneshot): add --usage-file JSON usage report to hermes -z Pipelines driving hermes -z (batch reviewers, cron scripts, eval harnesses) had no way to account for per-invocation spend: the agent computes estimated_cost_usd and full token counts internally, but oneshot mode discards everything except the final response text. - hermes -z PROMPT --usage-file PATH writes a JSON report after the run: estimated_cost_usd, cost_status/source, input/output/cache/ reasoning/total tokens, api_calls, model, provider, session_id, completed, failed. - Written even when the run fails (with a failure field) so callers can always account for spend; the write itself is best-effort and never masks the run's own outcome. - Flag registered in both the full parser and the Termux fast path; added to both value-flag scan sets so profile detection stays correct. Validation: 6 unit tests + live E2E (real -z run produced a report with real OpenRouter cost + token counts). * test: include usage_file kwarg in oneshot dispatch assertions The two dispatch tests assert the exact kwargs dict passed to run_oneshot; the new usage_file kwarg must appear there. --- hermes_cli/_parser.py | 11 ++++ hermes_cli/main.py | 4 ++ hermes_cli/oneshot.py | 47 ++++++++++++++ tests/hermes_cli/test_oneshot_usage_file.py | 69 +++++++++++++++++++++ tests/hermes_cli/test_tui_resume_flow.py | 2 + 5 files changed, 133 insertions(+) create mode 100644 tests/hermes_cli/test_oneshot_usage_file.py diff --git a/hermes_cli/_parser.py b/hermes_cli/_parser.py index d06a3d4ac9c..b2be73871b2 100644 --- a/hermes_cli/_parser.py +++ b/hermes_cli/_parser.py @@ -112,6 +112,17 @@ def build_top_level_parser(): "auto-bypassed. Intended for scripts / pipes." ), ) + parser.add_argument( + "--usage-file", + metavar="PATH", + default=None, + help=( + "One-shot mode only: after the run, write a JSON usage report " + "(estimated cost, token counts, model, api_calls) to PATH. " + "The report is written even when the run fails, so pipelines " + "can always account for spend. No effect outside -z/--oneshot." + ), + ) # --model / --provider are accepted at the top level so they can pair # with -z without needing the `chat` subcommand. If neither -z nor a # subcommand consumes them, they fall through harmlessly as None. diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 1c8cffd4c47..3434a39fa15 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -399,6 +399,7 @@ def _apply_profile_override() -> None: "-t", "--toolsets", "-r", "--resume", "-s", "--skills", + "--usage-file", } optional_value_flags = {"-c", "--continue"} i = 0 @@ -12240,6 +12241,7 @@ _TOP_LEVEL_VALUE_FLAGS = frozenset( "-t", "--toolsets", "-r", "--resume", "-s", "--skills", + "--usage-file", # ``-c / --continue`` is nargs='?' (optional value). Treat it as # value-taking: if the next token is a subcommand-looking word # the user almost certainly meant it as the session name, and @@ -12463,6 +12465,7 @@ def _try_termux_fast_cli_launch() -> bool: model=getattr(args, "model", None), provider=getattr(args, "provider", None), toolsets=getattr(args, "toolsets", None), + usage_file=getattr(args, "usage_file", None), ) ) @@ -14112,6 +14115,7 @@ def main(): model=getattr(args, "model", None), provider=getattr(args, "provider", None), toolsets=getattr(args, "toolsets", None), + usage_file=getattr(args, "usage_file", None), ) ) diff --git a/hermes_cli/oneshot.py b/hermes_cli/oneshot.py index 38d8e5e84bc..a4c6ca9cc25 100644 --- a/hermes_cli/oneshot.py +++ b/hermes_cli/oneshot.py @@ -25,6 +25,7 @@ import logging import os import sys from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path from typing import Optional from hermes_cli.fallback_config import get_fallback_chain @@ -122,11 +123,49 @@ def _validate_explicit_toolsets(toolsets: object = None) -> tuple[list[str] | No return valid, None +def _write_usage_file(path: Optional[str], result: dict, failure: Optional[str] = None) -> None: + """Best-effort JSON usage report for pipelines (``-z --usage-file``). + + Written even on failure so callers can always account for spend. Never + raises — a broken usage write must not mask the run's own outcome. + """ + if not path: + return + try: + import json + + report = { + "estimated_cost_usd": result.get("estimated_cost_usd"), + "cost_status": result.get("cost_status"), + "cost_source": result.get("cost_source"), + "input_tokens": result.get("input_tokens"), + "output_tokens": result.get("output_tokens"), + "cache_read_tokens": result.get("cache_read_tokens"), + "cache_write_tokens": result.get("cache_write_tokens"), + "reasoning_tokens": result.get("reasoning_tokens"), + "total_tokens": result.get("total_tokens"), + "api_calls": result.get("api_calls"), + "model": result.get("model"), + "provider": result.get("provider"), + "session_id": result.get("session_id"), + "completed": result.get("completed"), + "failed": bool(result.get("failed")) or failure is not None, + } + if failure is not None: + report["failure"] = failure + out = Path(path).expanduser() + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + except Exception: + pass + + def run_oneshot( prompt: str, model: Optional[str] = None, provider: Optional[str] = None, toolsets: object = None, + usage_file: Optional[str] = None, ) -> int: """Execute a single prompt and print only the final content block. @@ -137,6 +176,10 @@ def run_oneshot( provider: Optional provider override. Falls back to config.yaml's model.provider, then "auto". toolsets: Optional comma-separated string or iterable of toolsets. + usage_file: Optional path; when set, a JSON usage report (estimated + cost, token counts, model, api_calls) is written there after the + run — even when the run fails — so pipelines can account for + spend per invocation. Returns the exit code. Caller should sys.exit() with the return. """ @@ -209,11 +252,15 @@ def run_oneshot( # Re-raise control-flow exceptions so the parent handles them as usual # (Ctrl-C / explicit sys.exit() inside the agent). if isinstance(failure, (KeyboardInterrupt, SystemExit)): + _write_usage_file(usage_file, result, failure=repr(failure)) raise failure + _write_usage_file(usage_file, result, failure=str(failure)) real_stderr.write(f"hermes -z: agent failed: {failure}\n") real_stderr.flush() return 1 + _write_usage_file(usage_file, result) + if response: real_stdout.write(response) if not response.endswith("\n"): diff --git a/tests/hermes_cli/test_oneshot_usage_file.py b/tests/hermes_cli/test_oneshot_usage_file.py new file mode 100644 index 00000000000..3bee96cbbdb --- /dev/null +++ b/tests/hermes_cli/test_oneshot_usage_file.py @@ -0,0 +1,69 @@ +"""Tests for hermes -z --usage-file (per-run JSON usage report).""" + +import json + +from hermes_cli.oneshot import _write_usage_file + + +def _result(**overrides): + base = { + "estimated_cost_usd": 0.1234, + "cost_status": "estimated", + "cost_source": "pricing-table", + "input_tokens": 1000, + "output_tokens": 200, + "cache_read_tokens": 800, + "cache_write_tokens": 0, + "reasoning_tokens": 50, + "total_tokens": 1250, + "api_calls": 3, + "model": "openai/gpt-5.5", + "provider": "openrouter", + "session_id": "abc123", + "completed": True, + "failed": False, + } + base.update(overrides) + return base + + +class TestWriteUsageFile: + def test_writes_report_with_cost_and_tokens(self, tmp_path): + path = tmp_path / "usage.json" + _write_usage_file(str(path), _result()) + report = json.loads(path.read_text()) + assert report["estimated_cost_usd"] == 0.1234 + assert report["input_tokens"] == 1000 + assert report["output_tokens"] == 200 + assert report["model"] == "openai/gpt-5.5" + assert report["api_calls"] == 3 + assert report["failed"] is False + assert "failure" not in report + + def test_none_path_is_noop(self, tmp_path): + # Must not raise and must not create a report file. + _write_usage_file(None, _result()) + assert not (tmp_path / "usage.json").exists() + + def test_failure_marks_failed_and_records_message(self, tmp_path): + path = tmp_path / "usage.json" + _write_usage_file(str(path), {}, failure="boom") + report = json.loads(path.read_text()) + assert report["failed"] is True + assert report["failure"] == "boom" + # Missing result fields serialize as null, not KeyError. + assert report["estimated_cost_usd"] is None + + def test_creates_parent_directories(self, tmp_path): + path = tmp_path / "nested" / "dir" / "usage.json" + _write_usage_file(str(path), _result()) + assert json.loads(path.read_text())["total_tokens"] == 1250 + + def test_unwritable_path_never_raises(self): + # Root-owned path — the write must be swallowed, not raised. + _write_usage_file("/proc/definitely/not/writable/usage.json", _result()) + + def test_result_failed_flag_carries_through(self, tmp_path): + path = tmp_path / "usage.json" + _write_usage_file(str(path), _result(failed=True)) + assert json.loads(path.read_text())["failed"] is True diff --git a/tests/hermes_cli/test_tui_resume_flow.py b/tests/hermes_cli/test_tui_resume_flow.py index 2b6305ac8a3..34de95b826d 100644 --- a/tests/hermes_cli/test_tui_resume_flow.py +++ b/tests/hermes_cli/test_tui_resume_flow.py @@ -379,6 +379,7 @@ def test_termux_fast_cli_launch_oneshot_uses_light_parser(monkeypatch, main_mod) "model": "gpt-test", "provider": "openai", "toolsets": None, + "usage_file": None, } @@ -617,6 +618,7 @@ def test_main_top_level_oneshot_accepts_toolsets(monkeypatch, main_mod): "model": None, "provider": None, "toolsets": "web,terminal", + "usage_file": None, }