diff --git a/tests/tools/test_url_safety.py b/tests/tools/test_url_safety.py index 1745d98d026..b3386f092a6 100644 --- a/tests/tools/test_url_safety.py +++ b/tests/tools/test_url_safety.py @@ -43,6 +43,30 @@ class TestNormalizeUrlForRequest: == "https://xn--mnich-kva.example/K%C3%B6ln" ) + def test_repairs_space_between_scheme_and_authority(self): + assert ( + normalize_url_for_request("https:// docs.openclaw.ai") + == "https://docs.openclaw.ai" + ) + + def test_repairs_tab_between_scheme_and_authority(self): + assert ( + normalize_url_for_request("https:// docs.openclaw.ai/path") + == "https://docs.openclaw.ai/path" + ) + + def test_trims_but_preserves_path_and_query_space_semantics(self): + assert ( + normalize_url_for_request(" https://example.com/a b?q=c d ") + == "https://example.com/a%20b?q=c%20d" + ) + + def test_does_not_collapse_embedded_scheme_separator_in_query(self): + assert ( + normalize_url_for_request("https://example.com/r?next=https:// evil.example") + == "https://example.com/r?next=https://%20evil.example" + ) + class TestIsSafeUrl: def test_public_url_allowed(self): diff --git a/tools/url_safety.py b/tools/url_safety.py index e81f1061198..d7a7d125ef3 100644 --- a/tools/url_safety.py +++ b/tools/url_safety.py @@ -28,6 +28,7 @@ import logging import os import socket import asyncio +import re from typing import Any, Optional from urllib.parse import parse_qsl, quote, unquote, urljoin, urlparse, urlsplit, urlunsplit @@ -52,6 +53,13 @@ def normalize_url_for_request(url: str) -> str: if not raw: return raw + # Models sometimes emit otherwise valid URLs with whitespace between the + # scheme separator and authority (``https:// docs.example``). That position + # is never meaningful in HTTP(S) URLs, and repairing it before parsing keeps + # web tools from failing on a formatting artifact while leaving path/query + # whitespace to the normal percent-encoding path below. + raw = re.sub(r"^([A-Za-z][A-Za-z0-9+.-]*://)\s+", r"\1", raw) + try: parsed = urlsplit(raw) except ValueError: