diff --git a/agent/redact.py b/agent/redact.py index 06a7300a307..4e17ca50fa6 100644 --- a/agent/redact.py +++ b/agent/redact.py @@ -107,12 +107,60 @@ _PREFIX_PATTERNS = [ r"ntn_[A-Za-z0-9]{10,}", # Notion internal integration token ] -# ENV assignment patterns: KEY=value where KEY contains a secret-like name +# ENV assignment patterns: KEY=value where KEY contains a secret-like name. +# Uppercase keys tolerate spaces around "=" (e.g. ``FOO_SECRET = bar``) because +# an all-caps key is almost never prose/code. _SECRET_ENV_NAMES = r"(?:API_?KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|AUTH)" _ENV_ASSIGN_RE = re.compile( rf"([A-Z0-9_]{{0,50}}{_SECRET_ENV_NAMES}[A-Z0-9_]{{0,50}})\s*=\s*(['\"]?)(\S+)\2", ) +# Lowercase / dotted / hyphenated config keys from config files +# (application.properties, .env, YAML-ish dumps): ``spring.datasource.password=secret``, +# ``app.api.key=xyz``, ``password=secret``. The uppercase _ENV_ASSIGN_RE above +# never matched these, so config-file passwords leaked verbatim (issue #16413). +# +# These run only in a config-file context, NOT in prose, code, or URLs — three +# carve-outs preserved from the original design (#4367 + the documented +# web-URL passthrough below): +# 1. The value is bounded by ``[^\s&]`` (stops at whitespace AND ``&``) so +# form-urlencoded bodies are handled pair-by-pair (by _redact_form_body), +# not greedily swallowed. +# 2. _CFG_DOTTED_RE only matches when the key is NAMESPACED (contains a dot), +# which is unambiguously a config key — never a prose word. +# 3. _CFG_ANCHORED_RE matches a bare secret-word key only at line start +# (optionally after ``export``), so conversational ``I have password=foo`` +# mid-sentence is left alone. +# The colon-form URL guard (skip when ``://`` present) lives at the call site. +_SECRET_CFG_NAMES = r"(?:api[ _.\-]?key|token|secret|passwd|password|credential|auth)" +_CFG_VALUE = r"(['\"]?)([^\s&]+?)\2(?=[\s&]|$)" +# Namespaced (dotted) key: the secret word may sit anywhere in a dotted path. +_CFG_DOTTED_RE = re.compile( + rf"((?:[A-Za-z0-9_\-]+\.)+[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*" + rf"|[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*\.[A-Za-z0-9_.\-]+)" + rf"={_CFG_VALUE}", + re.IGNORECASE, +) +# Line-anchored bare key: ``password=…`` / ``export api_key=…`` at start of line. +_CFG_ANCHORED_RE = re.compile( + rf"(^[ \t]*(?:export[ \t]+)?[A-Za-z0-9_\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_\-]*)={_CFG_VALUE}", + re.IGNORECASE | re.MULTILINE, +) + +# Unquoted YAML / colon config (e.g. ``password: secret``, +# ``spring.datasource.password: hunter2``). The secret keyword must be part of +# the KEY (anchored to the start of the line/indent), and the value is a single +# whitespace-free token — so prose like ``note: secret meeting`` (keyword in the +# value) and ``error: token expired`` are left alone. Bare ``auth`` is excluded +# from the key set so ``Authorization:`` / ``author:`` don't match (the former +# is masked by _AUTH_HEADER_RE); ``auth_token``/``auth-token`` still match via +# the ``token`` keyword. Quoted values defer to _JSON_FIELD_RE via the lookahead. +_YAML_CFG_NAMES = r"(?:api[ _.\-]?key|token|secret|passwd|password|credential)" +_YAML_ASSIGN_RE = re.compile( + rf"(^[ \t]*[A-Za-z0-9_.\-]*{_YAML_CFG_NAMES}[A-Za-z0-9_.\-]*)(:[ \t]*)(?!['\"])([^\s&]+)", + re.IGNORECASE | re.MULTILINE, +) + # JSON field patterns: "apiKey": "value", "token": "value", etc. _JSON_KEY_NAMES = r"(?:api_?[Kk]ey|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)" _JSON_FIELD_RE = re.compile( @@ -382,6 +430,13 @@ def redact_sensitive_text(text: str, *, force: bool = False, code_file: bool = F name, quote, value = m.group(1), m.group(2), m.group(3) return f"{name}={quote}{_mask_token(value)}{quote}" text = _ENV_ASSIGN_RE.sub(_redact_env, text) + # Lowercase/dotted config keys (issue #16413). Skip URLs entirely — + # web-URL query params are intentionally passed through (see note + # near the bottom of this function); _DB_CONNSTR_RE still guards + # connection-string passwords. + if "://" not in text: + text = _CFG_DOTTED_RE.sub(_redact_env, text) + text = _CFG_ANCHORED_RE.sub(_redact_env, text) # JSON fields: "apiKey": "***" (skip for code files — false positives) if ":" in text and '"' in text: @@ -390,6 +445,15 @@ def redact_sensitive_text(text: str, *, force: bool = False, code_file: bool = F return f'{key}: "{_mask_token(value)}"' text = _JSON_FIELD_RE.sub(_redact_json, text) + # Unquoted YAML / colon config: password: *** (after JSON so quoted + # values are handled there; the lookahead in _YAML_ASSIGN_RE skips + # quotes). Skip URLs — web-URL query params pass through by design. + if ":" in text and "://" not in text: + def _redact_yaml(m): + key, sep, value = m.group(1), m.group(2), m.group(3) + return f"{key}{sep}{_mask_token(value)}" + text = _YAML_ASSIGN_RE.sub(_redact_yaml, text) + # Authorization headers — _AUTH_HEADER_RE matches any scheme after # "[Proxy-]Authorization:" case-insensitively, so "uthorization" is the # cheapest substring gate that covers every casing without a casefold(). diff --git a/tests/agent/test_redact.py b/tests/agent/test_redact.py index 88cc424a758..1423cc40ebb 100644 --- a/tests/agent/test_redact.py +++ b/tests/agent/test_redact.py @@ -504,6 +504,81 @@ class TestFormBodyRedaction: assert "first=1" in redact_sensitive_text(text) +class TestLowercaseDottedConfigKeys: + """Issue #16413 — config-file passwords in lowercase/dotted/colon keys + must be redacted. The uppercase _ENV_ASSIGN_RE missed these, leaking + `spring.datasource.password=...` and `password: ...` from `cat`'d config + files. Carve-outs: prose, code (#4367), and web URLs are left untouched. + """ + + def test_spring_dotted_password_assignment(self): + text = "spring.datasource.password=Sup3rS3cret!" + result = redact_sensitive_text(text) + assert "Sup3rS3cret!" not in result + assert "spring.datasource.password=" in result + + def test_dotted_api_key_split_keyword(self): + # 'api.key' splits the keyword across a dot — must still match. + text = "app.api.key=ak_live_998877" + result = redact_sensitive_text(text) + assert "ak_live_998877" not in result + assert "app.api.key=" in result + + def test_bare_lowercase_password_at_line_start(self): + text = "password=mysecretvalue123" + result = redact_sensitive_text(text) + assert "mysecretvalue123" not in result + + def test_quoted_lowercase_value(self): + text = "password='mysecretvalue123'" + result = redact_sensitive_text(text) + assert "mysecretvalue123" not in result + + def test_yaml_unquoted_password(self): + text = "password: Sup3rS3cret!" + result = redact_sensitive_text(text) + assert "Sup3rS3cret!" not in result + assert "password:" in result + + def test_yaml_indented_dotted(self): + text = "spring:\n datasource:\n password: hunter2pass" + result = redact_sensitive_text(text) + assert "hunter2pass" not in result + + def test_properties_file_dump(self): + text = ( + "server.port=8080\n" + "spring.datasource.username=admin\n" + "spring.datasource.password=Sup3rS3cret!\n" + "logging.level.root=INFO" + ) + result = redact_sensitive_text(text) + assert "Sup3rS3cret!" not in result + assert "server.port=8080" in result # non-secret keys preserved + assert "username=admin" in result + + # --- carve-outs: must NOT redact --- + + def test_prose_mid_sentence_password_unchanged(self): + # Not line-anchored, not dotted → conversational text, leave alone. + text = "I have password=foo and other things" + assert redact_sensitive_text(text) == text + + def test_lowercase_code_assignment_unchanged(self): + # #4367 regression — spaces around '=' in code. + text = "const secret = await fetchSecret();" + assert redact_sensitive_text(text) == text + + def test_url_query_param_passes_through(self): + # Web URLs are intentionally hands-off (documented design). + text = "https://example.com/api?password=opaqueval123&format=json" + assert redact_sensitive_text(text) == text + + def test_prose_keyword_in_value_unchanged(self): + text = "note: secret meeting at noon" + assert redact_sensitive_text(text) == text + + class TestXaiToken: KEY = "xai-ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstu"