perf: pre-compute sorted reasoning timeout floors at module level

_match_any() was re-sorting _REASONING_STALE_TIMEOUT_FLOORS (21 elements)
on every call. This function runs per API turn via error_classifier,
chat_completion_helpers, and thinking_timeout_guidance.

Also fixes thread-safety: the old _PATTERN_CACHE was a mutable dict
accessed from multiple threads without locking. Pre-compiling all
patterns at module load time eliminates both the per-call sort and
the TOCTOU race condition. The resulting list is effectively
immutable after import, safe for free-threaded Python 3.13+.

(cherry picked from commit e8b006b853)
This commit is contained in:
Gabriel Stoltemberg 2026-07-01 02:33:00 -03:00 committed by kshitij
parent 43a9bd9e0b
commit abc2069f49

View file

@ -146,19 +146,16 @@ _REASONING_STALE_TIMEOUT_FLOORS: tuple[tuple[str, int], ...] = (
# so we accept that community forks inheriting the same prefix are
# treated as reasoning models (a reasonable default — the upstream
# gateway timing is the same).
_PATTERN_CACHE: dict[str, re.Pattern[str]] = {}
def _get_pattern(slug: str) -> re.Pattern[str]:
compiled = _PATTERN_CACHE.get(slug)
if compiled is None:
compiled = re.compile(
r"^"
+ re.escape(slug)
+ r"(?:$|[\-._])"
)
_PATTERN_CACHE[slug] = compiled
return compiled
# Pre-compile all patterns at module load time to avoid per-call regex
# compilation and thread-safety issues with the mutable _PATTERN_CACHE.
# Tuples are immutable after creation, so this is safe for free-threaded
# Python 3.13+ without any locking.
_SORTED_REASONING_FLOORS: list[tuple[str, float, re.Pattern[str]]] = [
(slug, floor, re.compile(r"^" + re.escape(slug) + r"(?:$|[\-._])"))
for slug, floor in sorted(
_REASONING_STALE_TIMEOUT_FLOORS, key=lambda kv: -len(kv[0])
)
]
def _match_any(model_lower: str) -> Optional[float]:
@ -169,13 +166,8 @@ def _match_any(model_lower: str) -> Optional[float]:
order is irrelevant: longest slug wins (so ``o3-mini`` beats
``o3`` on a model like ``openai/o3-mini``).
"""
# Sort by slug length descending so longer / more-specific slugs
# win on shared prefixes (o3-mini beats o3).
sorted_floors = sorted(
_REASONING_STALE_TIMEOUT_FLOORS, key=lambda kv: -len(kv[0])
)
for slug, floor in sorted_floors:
if _get_pattern(slug).search(model_lower):
for _slug, floor, pattern in _SORTED_REASONING_FLOORS:
if pattern.search(model_lower):
return float(floor)
return None