mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-30 01:41:43 +00:00
37551ee53e
25 new tests (all Bedrock API calls mocked, no real AWS creds needed):
tests/hermes_cli/test_bedrock_model_picker.py (20 tests):
- provider_model_ids("bedrock") uses live discovery, returns regional
model IDs, falls back gracefully on empty/exception, resolves all
bedrock aliases (aws, aws-bedrock, amazon-bedrock) to live discovery
- list_authenticated_providers() section 2: bedrock appears with AWS
creds, model list from discover_bedrock_models(), total_models
matches, is_current flag works, absent creds hides bedrock, discovery
failure does not crash, no duplicate entries
- Region routing: botocore profile eu-central-1 yields eu.* model IDs
end-to-end; env var takes priority over botocore profile
- providers.py overlay: exists with correct transport/auth_type, label
is non-empty, all aliases normalize to bedrock
tests/agent/test_bedrock_adapter.py (5 tests):
- resolve_bedrock_region() botocore profile fallback, botocore failure
fallback, us-east-1 hard fallback (with botocore mocked)
324 lines
16 KiB
Python
324 lines
16 KiB
Python
"""Tests for AWS Bedrock integration in the model picker and provider catalog.
|
|
|
|
Covers the three paths changed by fix/bedrock-provider-model-ids-live-discovery:
|
|
|
|
1. provider_model_ids("bedrock") — uses live discover_bedrock_models() instead
|
|
of the static _PROVIDER_MODELS table, with curated fallback.
|
|
|
|
2. list_authenticated_providers() Section 2 (HERMES_OVERLAYS) — bedrock
|
|
appears when AWS credentials are present; model list comes from live
|
|
discovery keyed by the resolved region, NOT the static us.* table.
|
|
|
|
3. Region resolution — resolve_bedrock_region() reads from botocore profile
|
|
when no AWS_REGION / AWS_DEFAULT_REGION env vars are set, so EU/AP users
|
|
in eu-central-1 get eu.* profile IDs, not us.* ones.
|
|
|
|
All Bedrock API calls are mocked — no real AWS credentials needed.
|
|
"""
|
|
|
|
import os
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared helpers / fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_EU_MODELS = [
|
|
{"id": "eu.anthropic.claude-sonnet-4-6-20250514-v1:0", "name": "Claude Sonnet 4.6 (EU)", "provider": "inference-profile"},
|
|
{"id": "eu.anthropic.claude-haiku-4-5-20251015-v1:0", "name": "Claude Haiku 4.5 (EU)", "provider": "inference-profile"},
|
|
{"id": "eu.amazon.nova-pro-v1:0", "name": "Nova Pro (EU)", "provider": "inference-profile"},
|
|
]
|
|
|
|
_US_MODELS = [
|
|
{"id": "us.anthropic.claude-sonnet-4-6-20250514-v1:0", "name": "Claude Sonnet 4.6 (US)", "provider": "inference-profile"},
|
|
{"id": "us.amazon.nova-pro-v1:0", "name": "Nova Pro (US)", "provider": "inference-profile"},
|
|
]
|
|
|
|
|
|
def _mock_discover(region: str):
|
|
"""Return EU models for eu-* regions, US models otherwise."""
|
|
return _EU_MODELS if region.startswith("eu-") else _US_MODELS
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. provider_model_ids("bedrock")
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestProviderModelIdsBedrock:
|
|
"""provider_model_ids("bedrock") should use live Bedrock discovery."""
|
|
|
|
def test_returns_live_discovered_model_ids(self, monkeypatch):
|
|
"""Live discovery result is returned as a flat list of model ID strings."""
|
|
from hermes_cli.models import provider_model_ids
|
|
|
|
monkeypatch.setenv("AWS_REGION", "eu-central-1")
|
|
|
|
with patch("agent.bedrock_adapter.discover_bedrock_models", side_effect=_mock_discover), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
result = provider_model_ids("bedrock")
|
|
|
|
assert "eu.anthropic.claude-sonnet-4-6-20250514-v1:0" in result
|
|
assert "eu.anthropic.claude-haiku-4-5-20251015-v1:0" in result
|
|
assert len(result) == len(_EU_MODELS)
|
|
|
|
def test_region_determines_model_ids(self, monkeypatch):
|
|
"""Different regions produce different model ID prefixes (eu.* vs us.*)."""
|
|
from hermes_cli.models import provider_model_ids
|
|
|
|
with patch("agent.bedrock_adapter.discover_bedrock_models", side_effect=_mock_discover):
|
|
with patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
eu_result = provider_model_ids("bedrock")
|
|
with patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="us-east-1"):
|
|
us_result = provider_model_ids("bedrock")
|
|
|
|
assert all(m.startswith("eu.") for m in eu_result)
|
|
assert all(m.startswith("us.") for m in us_result)
|
|
assert eu_result != us_result
|
|
|
|
def test_falls_back_to_static_list_when_discovery_empty(self, monkeypatch):
|
|
"""When discover_bedrock_models() returns [], fall back to curated static list."""
|
|
from hermes_cli.models import _PROVIDER_MODELS, provider_model_ids
|
|
|
|
with patch("agent.bedrock_adapter.discover_bedrock_models", return_value=[]), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
result = provider_model_ids("bedrock")
|
|
|
|
# Should fall back to static table (may be empty or populated depending on
|
|
# the current static list, but must not crash and must be a list).
|
|
assert isinstance(result, list)
|
|
|
|
def test_falls_back_to_static_list_on_exception(self, monkeypatch):
|
|
"""When discover_bedrock_models() raises, fall back gracefully."""
|
|
from hermes_cli.models import provider_model_ids
|
|
|
|
with patch("agent.bedrock_adapter.discover_bedrock_models",
|
|
side_effect=Exception("boto3 not installed")), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
result = provider_model_ids("bedrock")
|
|
|
|
assert isinstance(result, list) # no crash
|
|
|
|
def test_accepts_bedrock_aliases(self, monkeypatch):
|
|
"""Provider aliases (aws, aws-bedrock, amazon) should also trigger live discovery."""
|
|
from hermes_cli.models import provider_model_ids
|
|
|
|
_expected_ids = [m["id"] for m in _US_MODELS]
|
|
|
|
with patch("agent.bedrock_adapter.discover_bedrock_models", side_effect=_mock_discover), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="us-east-1"):
|
|
for alias in ("aws", "aws-bedrock", "amazon-bedrock"):
|
|
result = provider_model_ids(alias)
|
|
assert result == _expected_ids, \
|
|
f"alias {alias!r} should return live-discovered US model IDs, got {result!r}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. list_authenticated_providers() — bedrock via HERMES_OVERLAYS (Section 2)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestListAuthenticatedProvidersBedrock:
|
|
"""Bedrock should appear in the /model picker when AWS creds are present."""
|
|
|
|
def test_bedrock_appears_with_aws_profile(self, monkeypatch):
|
|
"""Bedrock shows up when AWS_PROFILE is set."""
|
|
from hermes_cli.model_switch import list_authenticated_providers
|
|
|
|
monkeypatch.setenv("AWS_PROFILE", "my-sso-profile")
|
|
monkeypatch.setenv("AWS_REGION", "eu-central-1")
|
|
|
|
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
|
|
patch("agent.bedrock_adapter.discover_bedrock_models", side_effect=_mock_discover), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
providers = list_authenticated_providers(current_provider="bedrock")
|
|
|
|
bedrock = next((p for p in providers if p["slug"] == "bedrock"), None)
|
|
assert bedrock is not None, "bedrock should appear when AWS credentials are present"
|
|
|
|
def test_bedrock_uses_live_discovery_not_static_list(self, monkeypatch):
|
|
"""Model IDs come from discover_bedrock_models(), not the static _PROVIDER_MODELS table."""
|
|
from hermes_cli.model_switch import list_authenticated_providers
|
|
|
|
monkeypatch.setenv("AWS_PROFILE", "my-sso-profile")
|
|
|
|
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
|
|
patch("agent.bedrock_adapter.discover_bedrock_models", side_effect=_mock_discover), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
providers = list_authenticated_providers(current_provider="bedrock")
|
|
|
|
bedrock = next((p for p in providers if p["slug"] == "bedrock"), None)
|
|
assert bedrock is not None
|
|
|
|
# All returned model IDs should have eu.* prefix — live discovery result
|
|
for model_id in bedrock["models"]:
|
|
assert model_id.startswith("eu."), \
|
|
f"Expected eu.* model ID from live discovery, got {model_id!r}"
|
|
|
|
def test_bedrock_total_models_matches_discovery(self, monkeypatch):
|
|
"""total_models reflects the actual discovered count."""
|
|
from hermes_cli.model_switch import list_authenticated_providers
|
|
|
|
monkeypatch.setenv("AWS_PROFILE", "my-sso-profile")
|
|
|
|
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
|
|
patch("agent.bedrock_adapter.discover_bedrock_models", return_value=_EU_MODELS), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
providers = list_authenticated_providers(current_provider="openai")
|
|
|
|
bedrock = next((p for p in providers if p["slug"] == "bedrock"), None)
|
|
assert bedrock is not None
|
|
assert bedrock["total_models"] == len(_EU_MODELS)
|
|
|
|
def test_bedrock_is_current_when_selected(self, monkeypatch):
|
|
"""is_current=True when current_provider matches bedrock."""
|
|
from hermes_cli.model_switch import list_authenticated_providers
|
|
|
|
monkeypatch.setenv("AWS_PROFILE", "my-sso-profile")
|
|
|
|
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
|
|
patch("agent.bedrock_adapter.discover_bedrock_models", return_value=_EU_MODELS), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
providers = list_authenticated_providers(current_provider="bedrock")
|
|
|
|
bedrock = next((p for p in providers if p["slug"] == "bedrock"), None)
|
|
assert bedrock is not None
|
|
assert bedrock["is_current"] is True
|
|
|
|
def test_bedrock_not_shown_without_credentials(self, monkeypatch):
|
|
"""Bedrock must not appear when no AWS credentials are present."""
|
|
from hermes_cli.model_switch import list_authenticated_providers
|
|
|
|
monkeypatch.delenv("AWS_PROFILE", raising=False)
|
|
monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False)
|
|
monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False)
|
|
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
|
monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False)
|
|
monkeypatch.delenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", raising=False)
|
|
|
|
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=False):
|
|
providers = list_authenticated_providers(current_provider="openai")
|
|
|
|
bedrock = next((p for p in providers if p["slug"] == "bedrock"), None)
|
|
assert bedrock is None, "bedrock should NOT appear when AWS credentials are absent"
|
|
|
|
def test_bedrock_falls_back_to_curated_when_discovery_fails(self, monkeypatch):
|
|
"""When discover_bedrock_models() raises, fall back to curated list without crashing."""
|
|
from hermes_cli.model_switch import list_authenticated_providers
|
|
|
|
monkeypatch.setenv("AWS_PROFILE", "my-sso-profile")
|
|
|
|
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
|
|
patch("agent.bedrock_adapter.discover_bedrock_models",
|
|
side_effect=Exception("API call failed")), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
providers = list_authenticated_providers(current_provider="bedrock")
|
|
|
|
# Should not raise — bedrock entry may or may not appear depending on
|
|
# whether the curated fallback has entries, but the call must succeed.
|
|
assert isinstance(providers, list)
|
|
|
|
def test_bedrock_no_duplicate_entries(self, monkeypatch):
|
|
"""Bedrock must appear at most once — not in both Section 1 and Section 2."""
|
|
from hermes_cli.model_switch import list_authenticated_providers
|
|
|
|
monkeypatch.setenv("AWS_PROFILE", "my-sso-profile")
|
|
|
|
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
|
|
patch("agent.bedrock_adapter.discover_bedrock_models", return_value=_EU_MODELS), \
|
|
patch("agent.bedrock_adapter.resolve_bedrock_region", return_value="eu-central-1"):
|
|
providers = list_authenticated_providers(current_provider="bedrock")
|
|
|
|
bedrock_entries = [p for p in providers if p["slug"] == "bedrock"]
|
|
assert len(bedrock_entries) <= 1, \
|
|
f"bedrock should appear at most once, got {len(bedrock_entries)} entries"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. Region routing: EU/AP users see regional model IDs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestBedrockRegionRouting:
|
|
"""End-to-end: region from botocore profile is used for discovery, so EU/AP
|
|
users get eu.*/ap.* model IDs rather than the hardcoded us-east-1 list."""
|
|
|
|
def test_eu_region_from_botocore_profile_yields_eu_models(self):
|
|
"""When botocore resolves eu-central-1, picker shows eu.* model IDs."""
|
|
from hermes_cli.model_switch import list_authenticated_providers
|
|
|
|
mock_session = MagicMock()
|
|
mock_session.get_config_variable.return_value = "eu-central-1"
|
|
|
|
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
|
|
patch("agent.bedrock_adapter.discover_bedrock_models", side_effect=_mock_discover), \
|
|
patch("botocore.session.get_session", return_value=mock_session):
|
|
providers = list_authenticated_providers(current_provider="bedrock")
|
|
|
|
bedrock = next((p for p in providers if p["slug"] == "bedrock"), None)
|
|
assert bedrock is not None
|
|
for model_id in bedrock["models"]:
|
|
assert model_id.startswith("eu."), \
|
|
f"Expected eu.* model ID from eu-central-1 profile, got {model_id!r}"
|
|
|
|
def test_us_region_from_env_var_yields_us_models(self, monkeypatch):
|
|
"""Explicit AWS_REGION=us-east-1 returns us.* model IDs."""
|
|
from hermes_cli.model_switch import list_authenticated_providers
|
|
|
|
monkeypatch.setenv("AWS_REGION", "us-east-1")
|
|
|
|
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
|
|
patch("agent.bedrock_adapter.discover_bedrock_models", side_effect=_mock_discover):
|
|
providers = list_authenticated_providers(current_provider="bedrock")
|
|
|
|
bedrock = next((p for p in providers if p["slug"] == "bedrock"), None)
|
|
assert bedrock is not None
|
|
for model_id in bedrock["models"]:
|
|
assert model_id.startswith("us."), \
|
|
f"Expected us.* model ID from us-east-1, got {model_id!r}"
|
|
|
|
def test_env_var_takes_priority_over_botocore_profile(self, monkeypatch):
|
|
"""AWS_REGION env var wins over botocore profile region."""
|
|
from agent.bedrock_adapter import resolve_bedrock_region
|
|
|
|
monkeypatch.setenv("AWS_REGION", "us-west-2")
|
|
|
|
mock_session = MagicMock()
|
|
mock_session.get_config_variable.return_value = "eu-central-1"
|
|
|
|
with patch("botocore.session.get_session", return_value=mock_session):
|
|
region = resolve_bedrock_region()
|
|
|
|
assert region == "us-west-2", "env var should override botocore profile"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. providers.py overlay registration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestBedrockOverlayRegistration:
|
|
"""bedrock entry in HERMES_OVERLAYS is correctly configured."""
|
|
|
|
def test_bedrock_overlay_exists(self):
|
|
from hermes_cli.providers import HERMES_OVERLAYS
|
|
assert "bedrock" in HERMES_OVERLAYS
|
|
|
|
def test_bedrock_overlay_transport(self):
|
|
from hermes_cli.providers import HERMES_OVERLAYS
|
|
assert HERMES_OVERLAYS["bedrock"].transport == "bedrock_converse"
|
|
|
|
def test_bedrock_overlay_auth_type(self):
|
|
from hermes_cli.providers import HERMES_OVERLAYS
|
|
assert HERMES_OVERLAYS["bedrock"].auth_type == "aws_sdk"
|
|
|
|
def test_bedrock_label(self):
|
|
from hermes_cli.providers import get_label
|
|
label = get_label("bedrock")
|
|
assert label # non-empty
|
|
assert "bedrock" in label.lower() or "aws" in label.lower()
|
|
|
|
def test_bedrock_aliases_resolve(self):
|
|
from hermes_cli.providers import normalize_provider
|
|
for alias in ("aws", "aws-bedrock", "amazon-bedrock", "amazon"):
|
|
assert normalize_provider(alias) == "bedrock", \
|
|
f"alias {alias!r} should normalize to 'bedrock'"
|