From cf73b3d41101dd218f45bce80cd1b51e267b4489 Mon Sep 17 00:00:00 2001 From: arminanton <29869547+arminanton@users.noreply.github.com> Date: Tue, 23 Jun 2026 11:06:17 -0700 Subject: [PATCH] fix(copilot): clamp reasoning effort to the nearest supported level, not xhigh->high The Copilot provider profile unconditionally mapped ``xhigh`` to ``high`` before checking the model's catalog, so models that DO support ``xhigh`` (e.g. the gpt-5.x family per the live /models catalog) were silently capped one level down. Honor the requested effort when the catalog lists it as supported, and only downgrade when it does not, choosing the nearest weaker supported level (xhigh->high, minimal->low, else medium, else the first supported level). This matches the nearest-down clamp behavior used elsewhere for the ``max`` effort. Adds tests/plugins/model_providers/test_copilot_profile.py covering forward, downgrade, and fallback paths (catalog lookup stubbed). --- plugins/model-providers/copilot/__init__.py | 22 +++- .../model_providers/test_copilot_profile.py | 102 ++++++++++++++++++ 2 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 tests/plugins/model_providers/test_copilot_profile.py diff --git a/plugins/model-providers/copilot/__init__.py b/plugins/model-providers/copilot/__init__.py index f5c6ea865c4c..781a96c5ccb3 100644 --- a/plugins/model-providers/copilot/__init__.py +++ b/plugins/model-providers/copilot/__init__.py @@ -35,9 +35,25 @@ class CopilotProfile(ProviderProfile): supported_efforts = github_model_reasoning_efforts(model) if supported_efforts and reasoning_config: effort = reasoning_config.get("effort", "medium") - # Normalize stronger generic levels to the nearest supported. - if effort in {"xhigh", "max", "ultra"}: - effort = "high" + # Honor the requested level when the live Copilot catalog + # lists it as supported: gpt-5.5/gpt-5.4 DO support + # ``xhigh``. Only downgrade levels the catalog does NOT + # list (e.g. ``xhigh``/``max`` on models capped lower, or + # ``minimal`` where unsupported), choosing the nearest + # weaker supported level rather than forwarding verbatim. + # + # (Previously this unconditionally mapped xhigh->high, a + # stale guard that silently capped models which do support + # the higher level.) + if effort not in supported_efforts: + if effort == "xhigh" and "high" in supported_efforts: + effort = "high" + elif effort == "minimal" and "low" in supported_efforts: + effort = "low" + elif "medium" in supported_efforts: + effort = "medium" + else: + effort = supported_efforts[0] if effort in supported_efforts: extra_body["reasoning"] = {"effort": effort} elif supported_efforts: diff --git a/tests/plugins/model_providers/test_copilot_profile.py b/tests/plugins/model_providers/test_copilot_profile.py new file mode 100644 index 000000000000..bf1d3fc01b9e --- /dev/null +++ b/tests/plugins/model_providers/test_copilot_profile.py @@ -0,0 +1,102 @@ +"""Unit tests for the Copilot provider profile's reasoning-effort wiring. + +GitHub Copilot serves different models with different supported reasoning-effort +sets (the live ``/models`` catalog reports them per model). The profile must +forward the requested effort when the catalog lists it as supported, and only +downgrade to the nearest weaker supported level when it does not, rather than +unconditionally collapsing ``xhigh`` to ``high`` (which silently capped models +that actually support the higher level). + +These tests pin that contract without going live, by stubbing the catalog +lookup ``github_model_reasoning_efforts``. +""" + +from __future__ import annotations + +import pytest + + +@pytest.fixture +def copilot_profile(): + """Resolve the registered Copilot profile. + + Importing ``model_tools`` triggers plugin discovery, which registers the + Copilot profile. Going through ``get_provider_profile`` keeps the test + honest: if the registered class is ever swapped for a plain + ``ProviderProfile`` the assertions below collapse. + """ + import model_tools # noqa: F401 + import providers + + profile = providers.get_provider_profile("copilot") + assert profile is not None, "copilot provider profile must be registered" + return profile + + +def _patch_efforts(monkeypatch, efforts): + """Stub the catalog lookup the profile calls for supported efforts.""" + import hermes_cli.models as models_mod + monkeypatch.setattr( + models_mod, "github_model_reasoning_efforts", lambda model: list(efforts) + ) + + +class TestCopilotReasoningEffortClamp: + def test_supported_effort_forwarded_verbatim(self, copilot_profile, monkeypatch): + """xhigh is forwarded unchanged when the catalog lists it.""" + _patch_efforts(monkeypatch, ["minimal", "low", "medium", "high", "xhigh"]) + extra_body, _ = copilot_profile.build_api_kwargs_extras( + model="gpt-5.5", + reasoning_config={"effort": "xhigh"}, + supports_reasoning=True, + ) + assert extra_body["reasoning"] == {"effort": "xhigh"} + + def test_xhigh_downgrades_to_high_when_unsupported(self, copilot_profile, monkeypatch): + """A model whose catalog lacks xhigh gets the nearest weaker level.""" + _patch_efforts(monkeypatch, ["low", "medium", "high"]) + extra_body, _ = copilot_profile.build_api_kwargs_extras( + model="o-series-model", + reasoning_config={"effort": "xhigh"}, + supports_reasoning=True, + ) + assert extra_body["reasoning"] == {"effort": "high"} + + def test_minimal_downgrades_to_low_when_unsupported(self, copilot_profile, monkeypatch): + _patch_efforts(monkeypatch, ["low", "medium", "high"]) + extra_body, _ = copilot_profile.build_api_kwargs_extras( + model="o-series-model", + reasoning_config={"effort": "minimal"}, + supports_reasoning=True, + ) + assert extra_body["reasoning"] == {"effort": "low"} + + def test_unsupported_effort_falls_back_to_medium(self, copilot_profile, monkeypatch): + """An effort not in the set, with no specific rule, falls to medium.""" + _patch_efforts(monkeypatch, ["low", "medium", "high"]) + extra_body, _ = copilot_profile.build_api_kwargs_extras( + model="some-model", + reasoning_config={"effort": "garbage"}, + supports_reasoning=True, + ) + assert extra_body["reasoning"] == {"effort": "medium"} + + def test_falls_back_to_first_supported_when_no_medium(self, copilot_profile, monkeypatch): + """If medium isn't supported either, pick the first supported level.""" + _patch_efforts(monkeypatch, ["low", "high"]) + extra_body, _ = copilot_profile.build_api_kwargs_extras( + model="weird-model", + reasoning_config={"effort": "xhigh"}, + supports_reasoning=True, + ) + # xhigh not supported, high IS supported → high wins via the xhigh rule. + assert extra_body["reasoning"] == {"effort": "high"} + + def test_first_supported_when_no_rule_matches(self, copilot_profile, monkeypatch): + _patch_efforts(monkeypatch, ["low", "high"]) + extra_body, _ = copilot_profile.build_api_kwargs_extras( + model="weird-model", + reasoning_config={"effort": "garbage"}, + supports_reasoning=True, + ) + assert extra_body["reasoning"] == {"effort": "low"}