fix(browser): enforce cloud-metadata SSRF floor in hybrid routing (#16234) (#21228)

Cloud metadata endpoints (169.254.169.254 etc.) are now always blocked
by browser_navigate regardless of hybrid routing, allow_private_urls,
or backend.

Bug: commit 42c076d3 (#16136) added hybrid routing that flips
auto_local_this_nav=True for private URLs and short-circuits
_is_safe_url(). IMDS endpoints are technically private (169.254/16
link-local), so the sidecar happily routed them to a local Chromium,
and the agent could read IAM credentials via browser_snapshot. On
EC2/GCP/Azure this is a full SSRF-to-credential-theft.

Fix: new is_always_blocked_url() in url_safety.py — a narrow floor
that checks _BLOCKED_HOSTNAMES, _ALWAYS_BLOCKED_IPS,
_ALWAYS_BLOCKED_NETWORKS only. Applied as an independent gate in
browser_navigate's pre-nav and post-redirect checks, BEFORE
auto_local_this_nav gets a chance to short-circuit. Ordinary private
URLs (localhost, 192.168.x, 10.x, .local, CGNAT) still route to the
local sidecar as the #16136 feature intends.

Secondary fix (reporter's finding): _url_is_private() now explicitly
checks 172.16.0.0/12. ipaddress.is_private only covers that range on
Python ≥3.11 (bpo-40791), so on 3.10 runtimes those URLs were routed
to cloud instead of the local sidecar. No security impact — just a
correctness fix for the hybrid-routing feature.

Closes #16234.

tests/tools/test_url_safety.py

@@ -5,6 +5,7 @@ from unittest.mock import patch
 
 from tools.url_safety import (
     is_safe_url,
+    is_always_blocked_url,
     _is_blocked_ip,
     _global_allow_private_urls,
     _reset_allow_private_cache,
@@ -407,3 +408,69 @@ class TestAllowPrivateUrlsIntegration:
         """Empty URLs are still blocked."""
         monkeypatch.setenv("HERMES_ALLOW_PRIVATE_URLS", "true")
         assert is_safe_url("") is False
+
+
+class TestIsAlwaysBlockedUrl:
+    """The always-blocked floor — cloud metadata only, narrower than is_safe_url."""
+
+    # -- The sentinel set that must always block --------------------------------
+
+    @pytest.mark.parametrize("url", [
+        "http://169.254.169.254/latest/meta-data/",            # AWS / GCP / Azure / DO / Oracle
+        "http://169.254.169.253/metadata/instance",              # Azure IMDS wire server
+        "http://169.254.170.2/v2/credentials",                   # AWS ECS task metadata
+        "http://100.100.100.200/latest/meta-data/",              # Alibaba Cloud
+        "http://169.254.42.1/",                                  # Any /16 link-local
+    ])
+    def test_literal_imds_ips_always_blocked(self, url):
+        """Literal IMDS IPs and the /16 link-local range always block."""
+        assert is_always_blocked_url(url) is True
+
+    def test_gcp_metadata_hostname_always_blocked_even_without_dns(self):
+        """metadata.google.internal blocks by hostname, no DNS needed."""
+        with patch("socket.getaddrinfo", side_effect=socket.gaierror("nope")):
+            assert is_always_blocked_url("http://metadata.google.internal/") is True
+
+    def test_hostname_resolving_to_imds_always_blocked(self):
+        """Attacker-controlled hostname resolving to IMDS still blocks."""
+        with patch("socket.getaddrinfo", return_value=[
+            (2, 1, 6, "", ("169.254.169.254", 0)),
+        ]):
+            assert is_always_blocked_url("http://attacker-controlled.example.com/") is True
+
+    # -- Things the floor must NOT block ----------------------------------------
+
+    def test_public_url_not_blocked(self):
+        assert is_always_blocked_url("https://example.com/path") is False
+
+    @pytest.mark.parametrize("url", [
+        "http://127.0.0.1:8080/",
+        "http://192.168.1.1/",
+        "http://10.0.0.5/",
+        "http://172.16.0.1/",
+        "http://100.64.0.1/",  # CGNAT — blocked by is_safe_url but not by the floor
+    ])
+    def test_ordinary_private_urls_not_in_floor(self, url):
+        """Floor is narrower than is_safe_url — ordinary private URLs pass."""
+        assert is_always_blocked_url(url) is False
+
+    def test_dns_failure_not_in_floor(self):
+        """DNS failure on a non-sentinel hostname = not always-blocked.
+
+        Caller's ordinary fail-closed path (is_safe_url) handles that case.
+        """
+        with patch("socket.getaddrinfo", side_effect=socket.gaierror("fail")):
+            assert is_always_blocked_url("http://nonexistent.example.com/") is False
+
+    def test_empty_url_not_in_floor(self):
+        """Empty URL falls through — caller decides what to do with a malformed URL."""
+        assert is_always_blocked_url("") is False
+
+    def test_malformed_url_not_in_floor(self):
+        """Parse errors don't claim always-blocked status."""
+        assert is_always_blocked_url("not a url at all") is False
+
+    def test_floor_ignores_allow_private_urls_toggle(self, monkeypatch):
+        """security.allow_private_urls can NOT unblock cloud metadata."""
+        monkeypatch.setenv("HERMES_ALLOW_PRIVATE_URLS", "true")
+        assert is_always_blocked_url("http://169.254.169.254/") is True