mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
6b81590c55
Systematic prune per AGENTS.md test policy, one pass over every major test tree (gateway, hermes_cli, tools, agent, run_agent, plugins, cli, cron, tui_gateway, honcho/openviking, root-level): - DELETE: source-reading tests (read_text/getsource on prod files), change-detector tests (exact catalog counts, model-name snapshots, config version literals), mock-echo tests (assert a mock returns what it was told), assertion-free/trivial tests, near-duplicate parametrizations (boundaries + one representative kept), async/sync twin duplicates, cosmetic within-file variations. - KEEP (mandatory): security/redaction/approval guards, message-role alternation invariants, prompt-caching/deterministic-call-id invariants, issue-number regression tests (deduped), E2E tests. - 6 test files deleted outright (script-style/no-assert or fully redundant); conftest.py, fakes/, fixtures/ untouched. - tests/acp/conftest.py added: autouse fixture stubs the live models.dev/GitHub/Copilot/Anthropic inventory fetches that ACP server tests performed on every session create — test_server.py 147s → 3.4s, and the tests are now genuinely hermetic. - Sleep-based slowness shrunk where safe (codex_ttfb_watchdog, compression_concurrent_fork, etc.); no wall-clock assertion tightened. Verification: full hermetic suite via scripts/run_tests.sh — 2439 files, 31,130 tests passed, 0 failed, 0 flaky retries, 315s wall (baseline: 583s wall, 13,564s subprocess CPU).
72 lines
3 KiB
Python
72 lines
3 KiB
Python
"""Regression tests for two OpenAI/OpenRouter model-picker bugs.
|
|
|
|
Bug 1 — OpenAI picker dumped the raw ``/v1/models`` catalog
|
|
``provider_model_ids("openai")`` hit ``api.openai.com/v1/models`` and
|
|
returned the full 120+ entry catalog (embeddings, whisper, tts, dall-e,
|
|
moderation, gpt-3.5, …). The ``hermes model`` CLI shows only the curated
|
|
agentic list. The picker now intersects the live default-endpoint catalog
|
|
with the curated list (preserving curated order) so both surfaces match.
|
|
Custom OpenAI-compatible endpoints (proxies, gateways) keep the live list
|
|
verbatim so discovery still works.
|
|
|
|
Bug 2 — OpenRouter appeared authenticated whenever OPENAI_API_KEY was set
|
|
OpenRouter's HermesOverlay carried ``extra_env_vars=("OPENAI_API_KEY",)``.
|
|
``list_authenticated_providers`` reads ``extra_env_vars`` to decide whether
|
|
a provider has credentials, so any OpenAI user saw a phantom OpenRouter
|
|
row. The overlay entry is removed; runtime credential resolution still
|
|
falls back to OPENAI_API_KEY for explicitly-selected OpenRouter (handled
|
|
in runtime_provider.py, independent of the overlay).
|
|
"""
|
|
|
|
import os
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from hermes_cli import models as M
|
|
from hermes_cli.providers import HERMES_OVERLAYS
|
|
|
|
|
|
# --- Bug 2: overlay no longer lists OPENAI_API_KEY --------------------------
|
|
|
|
def test_openrouter_overlay_does_not_list_openai_api_key():
|
|
overlay = HERMES_OVERLAYS["openrouter"]
|
|
assert "OPENAI_API_KEY" not in overlay.extra_env_vars
|
|
|
|
|
|
# --- Bug 1: default OpenAI endpoint filters to curated agentic models -------
|
|
|
|
def test_default_openai_endpoint_filters_to_curated(monkeypatch):
|
|
"""The 126-model /v1/models dump is intersected with the curated list."""
|
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
|
|
|
curated = M._PROVIDER_MODELS["openai-api"]
|
|
# Live catalog: every curated model PLUS a pile of non-agentic junk.
|
|
live = list(curated) + [
|
|
"text-embedding-3-large", "whisper-1", "tts-1", "dall-e-3",
|
|
"gpt-3.5-turbo", "davinci-002", "omni-moderation-latest",
|
|
]
|
|
with patch.object(M, "fetch_api_models", return_value=live):
|
|
result = M.provider_model_ids("openai-api", force_refresh=True)
|
|
|
|
# Only curated models survive, in curated order, no junk.
|
|
assert result == list(curated)
|
|
for m in result:
|
|
assert m in curated
|
|
|
|
|
|
def test_default_openai_endpoint_intersects_account_access(monkeypatch):
|
|
"""Curated models the account can't access are dropped (intersection)."""
|
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
|
|
|
curated = M._PROVIDER_MODELS["openai-api"]
|
|
# Account only serves the first two curated models.
|
|
live = list(curated[:2]) + ["text-embedding-3-large", "whisper-1"]
|
|
with patch.object(M, "fetch_api_models", return_value=live):
|
|
result = M.provider_model_ids("openai-api", force_refresh=True)
|
|
|
|
assert result == list(curated[:2])
|
|
|
|
|