mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-25 00:51:20 +00:00
a1e667b9f2
Fix variable name breakage (run_agent, hermes_constants, etc.) where import rewriter changed 'import X' to 'import hermes_agent.Y' but test code still referenced 'X' as a variable name. Fix package-vs-module confusion (cli.auth, cli.models, cli.ui) where single files became directories. Fix hardcoded file paths in tests pointing to old locations. Fix tool registry to discover tools in subpackage directories. Fix stale import in hermes_agent/tools/__init__.py. Part of #14182, #14183
356 lines
15 KiB
Python
356 lines
15 KiB
Python
"""Tests for Google AI Studio (Gemini) provider integration."""
|
|
|
|
import os
|
|
import pytest
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
from hermes_agent.cli.auth.auth import PROVIDER_REGISTRY, resolve_provider, resolve_api_key_provider_credentials
|
|
from hermes_agent.cli.models.models import _PROVIDER_MODELS, _PROVIDER_LABELS, _PROVIDER_ALIASES, normalize_provider
|
|
from hermes_agent.cli.models.normalize import normalize_model_for_provider, detect_vendor
|
|
from hermes_agent.providers.metadata import get_model_context_length
|
|
from hermes_agent.providers.metadata_dev import PROVIDER_TO_MODELS_DEV, list_agentic_models, _NOISE_PATTERNS
|
|
|
|
|
|
# ── Provider Registry ──
|
|
|
|
class TestGeminiProviderRegistry:
|
|
def test_gemini_in_registry(self):
|
|
assert "gemini" in PROVIDER_REGISTRY
|
|
|
|
def test_gemini_config(self):
|
|
pconfig = PROVIDER_REGISTRY["gemini"]
|
|
assert pconfig.id == "gemini"
|
|
assert pconfig.name == "Google AI Studio"
|
|
assert pconfig.auth_type == "api_key"
|
|
assert pconfig.inference_base_url == "https://generativelanguage.googleapis.com/v1beta"
|
|
|
|
def test_gemini_env_vars(self):
|
|
pconfig = PROVIDER_REGISTRY["gemini"]
|
|
assert pconfig.api_key_env_vars == ("GOOGLE_API_KEY", "GEMINI_API_KEY")
|
|
assert pconfig.base_url_env_var == "GEMINI_BASE_URL"
|
|
|
|
def test_gemini_base_url(self):
|
|
assert "generativelanguage.googleapis.com" in PROVIDER_REGISTRY["gemini"].inference_base_url
|
|
|
|
|
|
# ── Provider Aliases ──
|
|
|
|
PROVIDER_ENV_VARS = (
|
|
"OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY",
|
|
"GOOGLE_API_KEY", "GEMINI_API_KEY", "GEMINI_BASE_URL",
|
|
"GLM_API_KEY", "ZAI_API_KEY", "KIMI_API_KEY",
|
|
"MINIMAX_API_KEY", "DEEPSEEK_API_KEY",
|
|
)
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clean_provider_env(monkeypatch):
|
|
for var in PROVIDER_ENV_VARS:
|
|
monkeypatch.delenv(var, raising=False)
|
|
|
|
|
|
class TestGeminiAliases:
|
|
def test_explicit_gemini(self):
|
|
assert resolve_provider("gemini") == "gemini"
|
|
|
|
def test_alias_google(self):
|
|
assert resolve_provider("google") == "gemini"
|
|
|
|
def test_alias_google_gemini(self):
|
|
assert resolve_provider("google-gemini") == "gemini"
|
|
|
|
def test_alias_google_ai_studio(self):
|
|
assert resolve_provider("google-ai-studio") == "gemini"
|
|
|
|
def test_models_py_aliases(self):
|
|
assert _PROVIDER_ALIASES.get("google") == "gemini"
|
|
assert _PROVIDER_ALIASES.get("google-gemini") == "gemini"
|
|
assert _PROVIDER_ALIASES.get("google-ai-studio") == "gemini"
|
|
|
|
def test_normalize_provider(self):
|
|
assert normalize_provider("google") == "gemini"
|
|
assert normalize_provider("gemini") == "gemini"
|
|
assert normalize_provider("google-ai-studio") == "gemini"
|
|
|
|
|
|
# ── Auto-detection ──
|
|
|
|
class TestGeminiAutoDetection:
|
|
def test_auto_detects_google_api_key(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "test-google-key")
|
|
assert resolve_provider("auto") == "gemini"
|
|
|
|
def test_auto_detects_gemini_api_key(self, monkeypatch):
|
|
monkeypatch.setenv("GEMINI_API_KEY", "test-gemini-key")
|
|
assert resolve_provider("auto") == "gemini"
|
|
|
|
def test_google_api_key_priority_over_gemini(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "primary-key")
|
|
monkeypatch.setenv("GEMINI_API_KEY", "alias-key")
|
|
creds = resolve_api_key_provider_credentials("gemini")
|
|
assert creds["api_key"] == "primary-key"
|
|
assert creds["source"] == "GOOGLE_API_KEY"
|
|
|
|
|
|
# ── Credential Resolution ──
|
|
|
|
class TestGeminiCredentials:
|
|
def test_resolve_with_google_api_key(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "google-secret")
|
|
creds = resolve_api_key_provider_credentials("gemini")
|
|
assert creds["provider"] == "gemini"
|
|
assert creds["api_key"] == "google-secret"
|
|
assert creds["base_url"] == "https://generativelanguage.googleapis.com/v1beta"
|
|
|
|
def test_resolve_with_gemini_api_key(self, monkeypatch):
|
|
monkeypatch.setenv("GEMINI_API_KEY", "gemini-secret")
|
|
creds = resolve_api_key_provider_credentials("gemini")
|
|
assert creds["api_key"] == "gemini-secret"
|
|
|
|
def test_resolve_with_custom_base_url(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "key")
|
|
monkeypatch.setenv("GEMINI_BASE_URL", "https://custom.endpoint/v1")
|
|
creds = resolve_api_key_provider_credentials("gemini")
|
|
assert creds["base_url"] == "https://custom.endpoint/v1"
|
|
|
|
def test_runtime_gemini(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "google-key")
|
|
from hermes_agent.cli.runtime_provider import resolve_runtime_provider
|
|
result = resolve_runtime_provider(requested="gemini")
|
|
assert result["provider"] == "gemini"
|
|
assert result["api_mode"] == "chat_completions"
|
|
assert result["api_key"] == "google-key"
|
|
assert result["base_url"] == "https://generativelanguage.googleapis.com/v1beta"
|
|
|
|
|
|
# ── Model Catalog ──
|
|
|
|
class TestGeminiModelCatalog:
|
|
def test_provider_entry_exists(self):
|
|
"""Gemini provider has a model catalog entry. Specific model names
|
|
are data that changes with Google releases and don't belong in tests.
|
|
"""
|
|
assert "gemini" in _PROVIDER_MODELS
|
|
assert len(_PROVIDER_MODELS["gemini"]) >= 1
|
|
|
|
def test_provider_label(self):
|
|
assert "gemini" in _PROVIDER_LABELS
|
|
assert _PROVIDER_LABELS["gemini"] == "Google AI Studio"
|
|
|
|
|
|
# ── Model Normalization ──
|
|
|
|
class TestGeminiModelNormalization:
|
|
def test_passthrough_bare_name(self):
|
|
assert normalize_model_for_provider("gemini-2.5-flash", "gemini") == "gemini-2.5-flash"
|
|
|
|
def test_strip_vendor_prefix(self):
|
|
assert normalize_model_for_provider("google/gemini-2.5-flash", "gemini") == "google/gemini-2.5-flash"
|
|
|
|
def test_gemma_vendor_detection(self):
|
|
assert detect_vendor("gemma-4-31b-it") == "google"
|
|
|
|
def test_gemini_vendor_detection(self):
|
|
assert detect_vendor("gemini-2.5-flash") == "google"
|
|
|
|
def test_aggregator_prepends_vendor(self):
|
|
result = normalize_model_for_provider("gemini-2.5-flash", "openrouter")
|
|
assert result == "google/gemini-2.5-flash"
|
|
|
|
def test_gemma_aggregator_prepends_vendor(self):
|
|
result = normalize_model_for_provider("gemma-4-31b-it", "openrouter")
|
|
assert result == "google/gemma-4-31b-it"
|
|
|
|
|
|
# ── Context Length ──
|
|
|
|
class TestGeminiContextLength:
|
|
def test_gemma_4_31b_context(self):
|
|
# Mock external API lookups to test against hardcoded defaults
|
|
# (models.dev and OpenRouter may return different values like 262144).
|
|
with patch("hermes_agent.providers.metadata_dev.lookup_models_dev_context", return_value=None), \
|
|
patch("hermes_agent.providers.metadata.fetch_model_metadata", return_value={}):
|
|
ctx = get_model_context_length("gemma-4-31b-it", provider="gemini")
|
|
assert ctx == 256000
|
|
|
|
def test_gemini_3_context(self):
|
|
ctx = get_model_context_length("gemini-3.1-pro-preview", provider="gemini")
|
|
assert ctx == 1048576
|
|
|
|
|
|
# ── Agent Init (no SyntaxError) ──
|
|
|
|
class TestGeminiAgentInit:
|
|
def test_agent_imports_without_error(self):
|
|
"""Verify run_agent.py has no SyntaxError (the critical bug)."""
|
|
import importlib
|
|
from hermes_agent.agent import loop as run_agent
|
|
importlib.reload(run_agent)
|
|
|
|
def test_gemini_agent_uses_chat_completions(self, monkeypatch):
|
|
"""Gemini still reports chat_completions even though the transport is native."""
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "test-key")
|
|
with patch("hermes_agent.providers.gemini_adapter.GeminiNativeClient") as mock_client:
|
|
mock_client.return_value = MagicMock()
|
|
from hermes_agent.agent.loop import AIAgent
|
|
agent = AIAgent(
|
|
model="gemini-2.5-flash",
|
|
provider="gemini",
|
|
api_key="test-key",
|
|
base_url="https://generativelanguage.googleapis.com/v1beta",
|
|
)
|
|
assert agent.api_mode == "chat_completions"
|
|
assert agent.provider == "gemini"
|
|
|
|
def test_gemini_agent_uses_native_client(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "AIzaSy_REAL_KEY")
|
|
with patch("hermes_agent.providers.gemini_adapter.GeminiNativeClient") as mock_client, \
|
|
patch("hermes_agent.agent.loop.OpenAI") as mock_openai, \
|
|
patch("hermes_agent.agent.loop.ContextCompressor") as mock_compressor:
|
|
mock_client.return_value = MagicMock()
|
|
mock_compressor.return_value = MagicMock(context_length=1048576, threshold_tokens=524288)
|
|
from hermes_agent.agent.loop import AIAgent
|
|
AIAgent(
|
|
model="gemini-2.5-flash",
|
|
provider="gemini",
|
|
api_key="AIzaSy_REAL_KEY",
|
|
base_url="https://generativelanguage.googleapis.com/v1beta",
|
|
)
|
|
assert mock_client.called
|
|
mock_openai.assert_not_called()
|
|
|
|
def test_gemini_custom_base_url_keeps_openai_client(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "AIzaSy_REAL_KEY")
|
|
with patch("hermes_agent.providers.gemini_adapter.GeminiNativeClient") as mock_client, \
|
|
patch("hermes_agent.agent.loop.OpenAI") as mock_openai, \
|
|
patch("hermes_agent.agent.loop.ContextCompressor") as mock_compressor:
|
|
mock_openai.return_value = MagicMock()
|
|
mock_compressor.return_value = MagicMock(context_length=128000, threshold_tokens=64000)
|
|
from hermes_agent.agent.loop import AIAgent
|
|
AIAgent(
|
|
model="gemini-2.5-flash",
|
|
provider="gemini",
|
|
api_key="AIzaSy_REAL_KEY",
|
|
base_url="https://proxy.example.com/v1",
|
|
)
|
|
mock_openai.assert_called_once()
|
|
|
|
def test_gemini_openai_compat_base_url_keeps_openai_client(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "AIzaSy_REAL_KEY")
|
|
with patch("hermes_agent.providers.gemini_adapter.GeminiNativeClient") as mock_client, \
|
|
patch("hermes_agent.agent.loop.OpenAI") as mock_openai, \
|
|
patch("hermes_agent.agent.loop.ContextCompressor") as mock_compressor:
|
|
mock_openai.return_value = MagicMock()
|
|
mock_compressor.return_value = MagicMock(context_length=1048576, threshold_tokens=524288)
|
|
from hermes_agent.agent.loop import AIAgent
|
|
AIAgent(
|
|
model="gemini-2.5-flash",
|
|
provider="gemini",
|
|
api_key="AIzaSy_REAL_KEY",
|
|
base_url="https://generativelanguage.googleapis.com/v1beta/openai",
|
|
)
|
|
mock_openai.assert_called_once()
|
|
|
|
def test_gemini_resolve_provider_client_uses_native_client(self, monkeypatch):
|
|
"""resolve_provider_client('gemini') should build GeminiNativeClient."""
|
|
monkeypatch.setenv("GEMINI_API_KEY", "AIzaSy_TEST_KEY")
|
|
with patch("hermes_agent.providers.gemini_adapter.GeminiNativeClient") as mock_client, \
|
|
patch("hermes_agent.providers.auxiliary.OpenAI") as mock_openai:
|
|
mock_client.return_value = MagicMock()
|
|
from hermes_agent.providers.auxiliary import resolve_provider_client
|
|
resolve_provider_client("gemini")
|
|
assert mock_client.called
|
|
mock_openai.assert_not_called()
|
|
|
|
def test_gemini_resolve_provider_client_keeps_openai_for_non_native_base_url(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_API_KEY", "AIzaSy_TEST_KEY")
|
|
monkeypatch.setenv("GEMINI_BASE_URL", "https://proxy.example.com/v1")
|
|
with patch("hermes_agent.providers.gemini_adapter.GeminiNativeClient") as mock_client, \
|
|
patch("hermes_agent.providers.auxiliary.OpenAI") as mock_openai:
|
|
mock_openai.return_value = MagicMock()
|
|
from hermes_agent.providers.auxiliary import resolve_provider_client
|
|
resolve_provider_client("gemini")
|
|
mock_openai.assert_called_once()
|
|
|
|
|
|
# ── models.dev Integration ──
|
|
|
|
class TestGeminiModelsDev:
|
|
def test_gemini_mapped_to_google(self):
|
|
assert PROVIDER_TO_MODELS_DEV.get("gemini") == "google"
|
|
|
|
def test_noise_filter_excludes_tts(self):
|
|
assert _NOISE_PATTERNS.search("gemini-2.5-pro-preview-tts")
|
|
|
|
def test_noise_filter_excludes_dated_preview(self):
|
|
assert _NOISE_PATTERNS.search("gemini-2.5-flash-preview-04-17")
|
|
|
|
def test_noise_filter_excludes_embedding(self):
|
|
assert _NOISE_PATTERNS.search("gemini-embedding-001")
|
|
|
|
def test_noise_filter_excludes_live(self):
|
|
assert _NOISE_PATTERNS.search("gemini-live-2.5-flash")
|
|
|
|
def test_noise_filter_excludes_image(self):
|
|
assert _NOISE_PATTERNS.search("gemini-2.5-flash-image")
|
|
|
|
def test_noise_filter_excludes_customtools(self):
|
|
assert _NOISE_PATTERNS.search("gemini-3.1-pro-preview-customtools")
|
|
|
|
def test_noise_filter_passes_stable(self):
|
|
assert not _NOISE_PATTERNS.search("gemini-2.5-flash")
|
|
|
|
def test_noise_filter_passes_preview(self):
|
|
# Non-dated preview (e.g. gemini-3-flash-preview) should pass
|
|
assert not _NOISE_PATTERNS.search("gemini-3-flash-preview")
|
|
|
|
def test_noise_filter_passes_gemma(self):
|
|
assert not _NOISE_PATTERNS.search("gemma-4-31b-it")
|
|
|
|
def test_list_agentic_models_with_mock_data(self):
|
|
"""list_agentic_models filters correctly from mock models.dev data."""
|
|
mock_data = {
|
|
"google": {
|
|
"models": {
|
|
"gemini-3-flash-preview": {"tool_call": True},
|
|
"gemini-2.5-pro": {"tool_call": True},
|
|
"gemini-embedding-001": {"tool_call": False},
|
|
"gemini-2.5-flash-preview-tts": {"tool_call": False},
|
|
"gemini-live-2.5-flash": {"tool_call": True},
|
|
"gemini-2.5-flash-preview-04-17": {"tool_call": True},
|
|
"gemma-4-31b-it": {"tool_call": True},
|
|
}
|
|
}
|
|
}
|
|
with patch("hermes_agent.providers.metadata_dev.fetch_models_dev", return_value=mock_data):
|
|
result = list_agentic_models("gemini")
|
|
assert "gemini-3-flash-preview" in result
|
|
assert "gemini-2.5-pro" in result
|
|
assert "gemma-4-31b-it" not in result
|
|
# Filtered out:
|
|
assert "gemini-embedding-001" not in result # no tool_call
|
|
assert "gemini-2.5-flash-preview-tts" not in result # no tool_call
|
|
assert "gemini-live-2.5-flash" not in result # noise: live-
|
|
assert "gemini-2.5-flash-preview-04-17" not in result # noise: dated preview
|
|
|
|
def test_list_provider_models_hides_low_tpm_google_gemmas(self):
|
|
mock_data = {
|
|
"google": {
|
|
"models": {
|
|
"gemini-2.5-pro": {},
|
|
"gemma-4-31b-it": {},
|
|
"gemma-3-27b-it": {},
|
|
"gemini-1.5-pro": {},
|
|
"gemini-2.0-flash": {},
|
|
}
|
|
}
|
|
}
|
|
with patch("hermes_agent.providers.metadata_dev.fetch_models_dev", return_value=mock_data):
|
|
from hermes_agent.providers.metadata_dev import list_provider_models
|
|
|
|
result = list_provider_models("gemini")
|
|
|
|
assert "gemini-2.5-pro" in result
|
|
assert "gemma-4-31b-it" not in result
|
|
assert "gemma-3-27b-it" not in result
|
|
assert "gemini-1.5-pro" not in result
|
|
assert "gemini-2.0-flash" not in result
|