From bc3f1f4f34aec277ef8faae4b956ef0c4c19ee50 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 24 May 2026 02:19:57 -0700 Subject: [PATCH] feat(secrets/bitwarden): EU Cloud + self-hosted server URL support (#31378) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #31370. bws defaults to the US identity endpoint, so EU Cloud and self-hosted machine-account tokens fail with [400 Bad Request] {"error":"invalid_client"} during 'hermes secrets bitwarden setup'. The token is valid — it's just being checked against the wrong region. Add a Bitwarden region step to the wizard between the access-token and project-list steps: Step 1 Install bws Step 2 Provide access token Step 3 Pick region <-- new (US / EU / self-hosted-custom-URL) Step 4 Pick project (now talks to the right endpoint) Step 5 Test fetch Region is stored in config.yaml as secrets.bitwarden.server_url and plumbed into every bws subprocess as BWS_SERVER_URL (project list, secret list, test fetch, and the env_loader startup pull). Also: - Non-interactive: 'hermes secrets bitwarden setup --server-url ...' - Pre-existing BWS_SERVER_URL in the shell is detected and reused - Cache key includes server_url so EU/US fetches don't collide - 'hermes secrets bitwarden status' shows the configured region - 'invalid_client' / '400 Bad Request' from bws now triggers a hint pointing at the region setting instead of looking like a bad token --- agent/secret_sources/bitwarden.py | 28 +++- hermes_cli/config.py | 8 ++ hermes_cli/env_loader.py | 1 + hermes_cli/secrets_cli.py | 142 ++++++++++++++++++- tests/test_bitwarden_secrets.py | 83 +++++++++++ website/docs/user-guide/secrets/bitwarden.md | 23 ++- 6 files changed, 271 insertions(+), 14 deletions(-) diff --git a/agent/secret_sources/bitwarden.py b/agent/secret_sources/bitwarden.py index fb6824b5229..8c1e8dc5678 100644 --- a/agent/secret_sources/bitwarden.py +++ b/agent/secret_sources/bitwarden.py @@ -70,7 +70,7 @@ _BWS_RUN_TIMEOUT = 30 # In-process cache so repeated load_hermes_dotenv() calls (CLI startup, # gateway hot-reload, test suites) don't re-fetch from BSM. -_CacheKey = Tuple[str, str] # (access_token_fingerprint, project_id) +_CacheKey = Tuple[str, str, str] # (access_token_fingerprint, project_id, server_url) _CACHE: Dict[_CacheKey, "_CachedFetch"] = {} @@ -317,11 +317,18 @@ def fetch_bitwarden_secrets( binary: Optional[Path] = None, cache_ttl_seconds: float = 300, use_cache: bool = True, + server_url: str = "", ) -> Tuple[Dict[str, str], List[str]]: """Pull the secrets for ``project_id`` from Bitwarden Secrets Manager. Returns ``(secrets_dict, warnings_list)``. + Set ``server_url`` to point at a non-default Bitwarden region or a + self-hosted instance — e.g. ``https://vault.bitwarden.eu`` for EU + Cloud accounts. When empty, ``bws`` uses its built-in default + (``https://vault.bitwarden.com``, US Cloud). This is plumbed into + the subprocess as ``BWS_SERVER_URL``. + Raises :class:`RuntimeError` for fatal conditions (missing binary, auth failure, unparseable output). Callers in the env_loader path catch this and emit a single warning; callers in the user-facing @@ -332,7 +339,7 @@ def fetch_bitwarden_secrets( if not project_id: raise RuntimeError("Bitwarden project_id is empty") - cache_key = (_token_fingerprint(access_token), project_id) + cache_key = (_token_fingerprint(access_token), project_id, server_url or "") if use_cache: cached = _CACHE.get(cache_key) if cached and cached.is_fresh(cache_ttl_seconds): @@ -347,19 +354,26 @@ def fetch_bitwarden_secrets( "`hermes secrets bitwarden setup`." ) - secrets, warnings = _run_bws_list(bws, access_token, project_id) + secrets, warnings = _run_bws_list(bws, access_token, project_id, server_url) _CACHE[cache_key] = _CachedFetch(secrets=secrets, fetched_at=time.time()) return secrets, warnings def _run_bws_list( - bws: Path, access_token: str, project_id: str + bws: Path, access_token: str, project_id: str, server_url: str = "" ) -> Tuple[Dict[str, str], List[str]]: cmd = [str(bws), "secret", "list", project_id, "--output", "json"] env = os.environ.copy() env["BWS_ACCESS_TOKEN"] = access_token # Make sure we're not echoing telemetry / colour codes into json. env.setdefault("NO_COLOR", "1") + # Region / self-hosted support. bws defaults to https://vault.bitwarden.com + # (US Cloud); EU Cloud users need https://vault.bitwarden.eu, and + # self-hosted users need their own URL. When unset, fall back to whatever + # BWS_SERVER_URL the caller already had in their shell env (preserved by + # the copy above) so manual overrides keep working too. + if server_url: + env["BWS_SERVER_URL"] = server_url try: proc = subprocess.run( # noqa: S603 — bws path is trusted @@ -437,6 +451,7 @@ def apply_bitwarden_secrets( override_existing: bool = False, cache_ttl_seconds: float = 300, auto_install: bool = True, + server_url: str = "", ) -> FetchResult: """Pull secrets from BSM and set them on ``os.environ``. @@ -444,6 +459,10 @@ def apply_bitwarden_secrets( files have loaded. It is intentionally defensive — any failure returns a :class:`FetchResult` with ``error`` set; it never raises. + ``server_url`` selects the Bitwarden region or self-hosted endpoint + (e.g. ``https://vault.bitwarden.eu`` for EU Cloud). Empty string + means use ``bws``'s default (US Cloud). + Parameters mirror the ``secrets.bitwarden.*`` config keys so the caller can just splat the dict in. """ @@ -482,6 +501,7 @@ def apply_bitwarden_secrets( project_id=project_id, binary=binary, cache_ttl_seconds=cache_ttl_seconds, + server_url=server_url, ) except RuntimeError as exc: result.error = str(exc) diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 715fd7eb76f..f3ee2632b86 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1775,6 +1775,14 @@ DEFAULT_CONFIG = { # ~/.hermes/bin/ on first use. When False you must install # bws yourself and have it on PATH. "auto_install": True, + # Bitwarden region / self-hosted endpoint. Empty string + # means use the bws CLI default (US Cloud, + # https://vault.bitwarden.com). Set to + # https://vault.bitwarden.eu for EU Cloud, or your own URL + # for self-hosted Bitwarden. Plumbed into the bws subprocess + # as BWS_SERVER_URL. Prompted for during + # `hermes secrets bitwarden setup`. + "server_url": "", }, }, diff --git a/hermes_cli/env_loader.py b/hermes_cli/env_loader.py index 0cc5f9ba3c0..40a87830dfe 100644 --- a/hermes_cli/env_loader.py +++ b/hermes_cli/env_loader.py @@ -252,6 +252,7 @@ def _apply_external_secret_sources(home_path: Path) -> None: override_existing=bool(bw_cfg.get("override_existing", False)), cache_ttl_seconds=float(bw_cfg.get("cache_ttl_seconds", 300)), auto_install=bool(bw_cfg.get("auto_install", True)), + server_url=str(bw_cfg.get("server_url", "") or "").strip(), ) if result.applied: diff --git a/hermes_cli/secrets_cli.py b/hermes_cli/secrets_cli.py index d771969017e..38a638576bd 100644 --- a/hermes_cli/secrets_cli.py +++ b/hermes_cli/secrets_cli.py @@ -57,6 +57,15 @@ def register_cli(parent_parser: argparse.ArgumentParser) -> None: "--access-token", help="Provide the access token non-interactively (will be stored in .env)", ) + setup.add_argument( + "--server-url", + help=( + "Bitwarden region / self-hosted endpoint. Examples: " + "https://vault.bitwarden.com (US, default), " + "https://vault.bitwarden.eu (EU), or your self-hosted URL. " + "Skips the interactive region prompt." + ), + ) setup.set_defaults(func=cmd_setup) status = sub.add_parser("status", help="Show config + binary + last fetch") @@ -145,14 +154,28 @@ def cmd_setup(args: argparse.Namespace) -> int: os.environ[token_env] = token # so the test fetch below sees it console.print(f" [green]✓[/green] stored in {get_env_path()} as {token_env}") + # ------------------------------------------------------------------ region + console.print() + console.print("[bold]Step 3[/bold] Pick a Bitwarden region") + server_url = _resolve_server_url(args, secrets_cfg, console) + if server_url is None: + return 1 + if server_url: + console.print(f" [green]✓[/green] using {server_url}") + else: + console.print( + " [green]✓[/green] using bws default " + "(US Cloud, https://vault.bitwarden.com)" + ) + # ------------------------------------------------------------------- project if args.project_id and args.project_id.strip(): project_id = args.project_id.strip() else: console.print() - console.print("[bold]Step 3[/bold] Pick a project") + console.print("[bold]Step 4[/bold] Pick a project") project_id = "" - projects = _list_projects(binary, token, console) + projects = _list_projects(binary, token, console, server_url=server_url) if projects is None: return 1 if not projects: @@ -187,7 +210,7 @@ def cmd_setup(args: argparse.Namespace) -> int: # ------------------------------------------------------------------- test console.print() - step_num = 4 if not (args.project_id and args.project_id.strip()) else 3 + step_num = 5 if not (args.project_id and args.project_id.strip()) else 4 console.print(f"[bold]Step {step_num}[/bold] Test fetch") try: secrets, warnings = bw.fetch_bitwarden_secrets( @@ -195,6 +218,7 @@ def cmd_setup(args: argparse.Namespace) -> int: project_id=project_id, binary=binary, use_cache=False, + server_url=server_url, ) except Exception as exc: # noqa: BLE001 console.print(f" [red]✗ Fetch failed: {exc}[/red]") @@ -221,6 +245,7 @@ def cmd_setup(args: argparse.Namespace) -> int: # ------------------------------------------------------------------- save secrets_cfg["enabled"] = True secrets_cfg["project_id"] = project_id + secrets_cfg["server_url"] = server_url secrets_cfg.setdefault("access_token_env", token_env) secrets_cfg.setdefault("cache_ttl_seconds", 300) secrets_cfg.setdefault("override_existing", True) @@ -248,6 +273,7 @@ def cmd_status(args: argparse.Namespace) -> int: enabled = bool(bw_cfg.get("enabled")) token_env = bw_cfg.get("access_token_env", "BWS_ACCESS_TOKEN") project_id = bw_cfg.get("project_id", "") + server_url = str(bw_cfg.get("server_url", "") or "").strip() token_set = bool(os.environ.get(token_env)) table = Table(show_header=False, box=None, padding=(0, 2)) @@ -257,6 +283,10 @@ def cmd_status(args: argparse.Namespace) -> int: table.add_row("Token env var", token_env) table.add_row("Token in env", _yn(token_set)) table.add_row("Project ID", project_id or "[dim](unset)[/dim]") + table.add_row( + "Server URL", + server_url or "[dim]default (US Cloud, https://vault.bitwarden.com)[/dim]", + ) table.add_row("Override existing", _yn(bool(bw_cfg.get("override_existing", False)))) table.add_row("Cache TTL (s)", str(bw_cfg.get("cache_ttl_seconds", 300))) table.add_row("Auto-install", _yn(bool(bw_cfg.get("auto_install", True)))) @@ -306,11 +336,14 @@ def cmd_sync(args: argparse.Namespace) -> int: console.print("[red]No project_id configured.[/red]") return 1 + server_url = str(bw_cfg.get("server_url", "") or "").strip() + try: secrets, warnings = bw.fetch_bitwarden_secrets( access_token=token, project_id=project_id, use_cache=False, + server_url=server_url, ) except Exception as exc: # noqa: BLE001 console.print(f"[red]Fetch failed: {exc}[/red]") @@ -407,12 +440,14 @@ def _bws_version(binary: Path) -> str: def _list_projects( - binary: Path, token: str, console: Console + binary: Path, token: str, console: Console, *, server_url: str = "" ) -> Optional[List[dict]]: """Call ``bws project list`` and return the parsed list, or None on failure.""" env = os.environ.copy() env["BWS_ACCESS_TOKEN"] = token env.setdefault("NO_COLOR", "1") + if server_url: + env["BWS_SERVER_URL"] = server_url try: res = subprocess.run( [str(binary), "project", "list", "--output", "json"], @@ -428,7 +463,16 @@ def _list_projects( if res.returncode != 0: err = (res.stderr or res.stdout).strip()[:300] console.print(f" [red]bws project list failed: {err}[/red]") - if "authorization" in err.lower() or "invalid" in err.lower(): + lowered = err.lower() + if "invalid_client" in lowered or "400 bad request" in lowered: + console.print( + " [yellow]'invalid_client' from the US identity endpoint usually " + "means the token is for a different Bitwarden region. Re-run " + "[cyan]hermes secrets bitwarden setup[/cyan] and pick EU or " + "self-hosted at the region prompt, or set [cyan]secrets.bitwarden." + "server_url[/cyan] in config.yaml.[/yellow]" + ) + elif "authorization" in lowered or "invalid" in lowered: console.print( " [yellow]This usually means the access token is wrong or revoked. " "Double-check it in the Bitwarden web app.[/yellow]" @@ -443,3 +487,91 @@ def _list_projects( if not isinstance(data, list): return [] return [p for p in data if isinstance(p, dict) and p.get("id")] + + +# Canonical Bitwarden region endpoints. Keep in sync with what Bitwarden +# publishes — these are stable but if a third region appears, add it here +# and to the prompt below. +_REGION_PRESETS = [ + ("US Cloud (https://vault.bitwarden.com — bws default)", ""), + ("EU Cloud (https://vault.bitwarden.eu)", "https://vault.bitwarden.eu"), +] + + +def _resolve_server_url( + args: argparse.Namespace, + secrets_cfg: dict, + console: Console, +) -> Optional[str]: + """Pick a Bitwarden server URL for setup. + + Resolution order: + 1. ``--server-url`` CLI flag (non-interactive) + 2. ``BWS_SERVER_URL`` env var (so users running with that already set + in their shell don't have to re-enter it) + 3. Existing ``secrets.bitwarden.server_url`` value (for re-runs) + 4. Interactive menu: US / EU / self-hosted + + Returns the chosen URL as a string (empty string = bws default, + i.e. US Cloud). Returns None if the user aborted with an empty + custom URL. + """ + if args.server_url and args.server_url.strip(): + return args.server_url.strip() + + env_url = os.environ.get("BWS_SERVER_URL", "").strip() + if env_url: + console.print( + f" Detected [cyan]BWS_SERVER_URL[/cyan]={env_url} in your shell — using it." + ) + return env_url + + existing = str(secrets_cfg.get("server_url", "") or "").strip() + if existing: + console.print( + f" Existing config: [cyan]{existing}[/cyan]. " + "Press Enter to keep, or pick a different option below." + ) + + table = Table(show_header=True, header_style="bold", box=None, padding=(0, 2)) + table.add_column("#", style="cyan", width=4) + table.add_column("Region / endpoint") + for i, (label, _url) in enumerate(_REGION_PRESETS, 1): + table.add_row(str(i), label) + table.add_row(str(len(_REGION_PRESETS) + 1), "Self-hosted / custom URL") + console.print(table) + + custom_idx = len(_REGION_PRESETS) + 1 + while True: + prompt = f" Select region [1-{custom_idx}]" + if existing: + prompt += " (Enter to keep current)" + prompt += ": " + choice = console.input(prompt).strip() + if not choice: + if existing: + return existing + console.print(" [red]Enter a number.[/red]") + continue + try: + idx = int(choice) + except ValueError: + console.print(" [red]Enter a number.[/red]") + continue + if 1 <= idx <= len(_REGION_PRESETS): + return _REGION_PRESETS[idx - 1][1] + if idx == custom_idx: + custom = console.input( + " Enter your Bitwarden server URL " + "(e.g. https://vault.example.com): " + ).strip() + if not custom: + console.print(" [red]Empty URL, aborting.[/red]") + return None + if not custom.startswith(("http://", "https://")): + console.print( + " [yellow]Warning: URL doesn't start with http:// or " + "https:// — bws may reject it.[/yellow]" + ) + return custom + console.print(f" [red]Out of range — pick 1-{custom_idx}.[/red]") diff --git a/tests/test_bitwarden_secrets.py b/tests/test_bitwarden_secrets.py index 47155795750..125fbcdc49e 100644 --- a/tests/test_bitwarden_secrets.py +++ b/tests/test_bitwarden_secrets.py @@ -301,6 +301,89 @@ def test_fetch_cache_hits(monkeypatch, tmp_path): assert call_count["n"] == 1 # cached on second call +def test_fetch_server_url_sets_env(monkeypatch, tmp_path): + """server_url must be plumbed into the subprocess as BWS_SERVER_URL.""" + fake_binary = tmp_path / "bws" + fake_binary.write_text("") + payload = _fake_bws_payload([{"key": "K", "value": "v"}]) + + captured_env = {} + + def fake_run(cmd, **kwargs): + captured_env.update(kwargs["env"]) + return mock.Mock(returncode=0, stdout=payload, stderr="") + + monkeypatch.setattr(bw.subprocess, "run", fake_run) + + bw.fetch_bitwarden_secrets( + access_token="0.t", + project_id="p", + binary=fake_binary, + use_cache=False, + server_url="https://vault.bitwarden.eu", + ) + assert captured_env.get("BWS_SERVER_URL") == "https://vault.bitwarden.eu" + + +def test_fetch_no_server_url_does_not_set_env(monkeypatch, tmp_path): + """When server_url is empty, BWS_SERVER_URL must not be injected.""" + fake_binary = tmp_path / "bws" + fake_binary.write_text("") + payload = _fake_bws_payload([]) + # Make sure the inherited env doesn't already have BWS_SERVER_URL set. + monkeypatch.delenv("BWS_SERVER_URL", raising=False) + + captured_env = {} + + def fake_run(cmd, **kwargs): + captured_env.update(kwargs["env"]) + return mock.Mock(returncode=0, stdout=payload, stderr="") + + monkeypatch.setattr(bw.subprocess, "run", fake_run) + + bw.fetch_bitwarden_secrets( + access_token="0.t", + project_id="p", + binary=fake_binary, + use_cache=False, + ) + assert "BWS_SERVER_URL" not in captured_env + + +def test_fetch_server_url_keyed_in_cache(monkeypatch, tmp_path): + """Different server_url values must produce separate cache entries.""" + fake_binary = tmp_path / "bws" + fake_binary.write_text("") + payload = _fake_bws_payload([{"key": "K", "value": "v"}]) + + call_count = {"n": 0} + + def fake_run(*a, **kw): + call_count["n"] += 1 + return mock.Mock(returncode=0, stdout=payload, stderr="") + + monkeypatch.setattr(bw.subprocess, "run", fake_run) + + # US (default empty) — fresh fetch. + bw.fetch_bitwarden_secrets( + access_token="0.t", project_id="p", + binary=fake_binary, cache_ttl_seconds=60, + ) + # EU — different server_url, must NOT hit the US cache entry. + bw.fetch_bitwarden_secrets( + access_token="0.t", project_id="p", + binary=fake_binary, cache_ttl_seconds=60, + server_url="https://vault.bitwarden.eu", + ) + # Second EU call hits cache. + bw.fetch_bitwarden_secrets( + access_token="0.t", project_id="p", + binary=fake_binary, cache_ttl_seconds=60, + server_url="https://vault.bitwarden.eu", + ) + assert call_count["n"] == 2 + + def test_fetch_cache_disabled(monkeypatch, tmp_path): fake_binary = tmp_path / "bws" fake_binary.write_text("") diff --git a/website/docs/user-guide/secrets/bitwarden.md b/website/docs/user-guide/secrets/bitwarden.md index 34d45a6038a..3e518512472 100644 --- a/website/docs/user-guide/secrets/bitwarden.md +++ b/website/docs/user-guide/secrets/bitwarden.md @@ -21,7 +21,7 @@ You set up the machine account *in the web app*, where your normal 2FA applies. ### 1. Create a machine account and access token -In the [Bitwarden web app](https://vault.bitwarden.com): +In the [Bitwarden web app](https://vault.bitwarden.com) (or [vault.bitwarden.eu](https://vault.bitwarden.eu) for EU accounts): 1. Switch to **Secrets Manager** from the product switcher. 2. Create or pick a **Project** (e.g. "Hermes keys"). @@ -41,9 +41,19 @@ It will: 1. Download and verify `bws v2.0.0` into `~/.hermes/bin/bws`. 2. Prompt you for the access token (input is hidden). Stored in `~/.hermes/.env` as `BWS_ACCESS_TOKEN`. -3. List the projects the machine account can see; pick one. Stored in `config.yaml` as `secrets.bitwarden.project_id`. -4. Test-fetch the project's secrets and show you which env vars will resolve. -5. Flip `secrets.bitwarden.enabled: true`. +3. Ask which Bitwarden region your machine account belongs to — **US Cloud**, **EU Cloud**, or **self-hosted / custom URL**. Stored in `config.yaml` as `secrets.bitwarden.server_url` and passed to `bws` as `BWS_SERVER_URL`. +4. List the projects the machine account can see; pick one. Stored in `config.yaml` as `secrets.bitwarden.project_id`. +5. Test-fetch the project's secrets and show you which env vars will resolve. +6. Flip `secrets.bitwarden.enabled: true`. + +Non-interactive setup is also supported via flags: + +```bash +hermes secrets bitwarden setup \ + --access-token "$BWS_ACCESS_TOKEN" \ + --server-url https://vault.bitwarden.eu \ + --project-id +``` ### 3. Confirm @@ -74,6 +84,7 @@ secrets: enabled: false access_token_env: BWS_ACCESS_TOKEN project_id: "" + server_url: "" cache_ttl_seconds: 300 override_existing: true auto_install: true @@ -84,6 +95,7 @@ secrets: | `enabled` | `false` | Master switch. When false, Bitwarden is never contacted. | | `access_token_env` | `BWS_ACCESS_TOKEN` | Env var name that holds the bootstrap token. Change this if you already use `BWS_ACCESS_TOKEN` for something else. | | `project_id` | `""` | UUID of the project to sync from. | +| `server_url` | `""` | Bitwarden region or self-hosted endpoint. Empty = `bws` default (US Cloud, `https://vault.bitwarden.com`). Set to `https://vault.bitwarden.eu` for EU Cloud, or your own URL for self-hosted. Plumbed into the `bws` subprocess as `BWS_SERVER_URL`. | | `cache_ttl_seconds` | `300` | How long an in-process fetch result is reused. Set to `0` to disable caching. Cache is per-process; new `hermes` invocations start fresh. | | `override_existing` | `true` | When true, Bitwarden values overwrite anything already in env (so rotation in the web app actually takes effect). Flip to `false` if you want `.env` / shell exports to win locally. | | `auto_install` | `true` | When true, `bws` is auto-downloaded into `~/.hermes/bin/` on first use. | @@ -96,7 +108,8 @@ Bitwarden never blocks Hermes startup. If anything goes wrong, you'll see a one- |---|---|---| | `BWS_ACCESS_TOKEN is not set` | Enabled in config but token cleared from `.env` | Re-run `hermes secrets bitwarden setup` | | `bws exited 1: invalid access token` | Token revoked or wrong | Generate a new token, re-run setup | -| `bws timed out` | Network blocked or Bitwarden API slow | Check connectivity to `api.bitwarden.com` | +| `[400 Bad Request] {"error":"invalid_client"}` | Token is for a Bitwarden region other than the one `bws` is calling (e.g. EU token hitting the US identity endpoint) | Re-run setup and pick the right region, or set `secrets.bitwarden.server_url` to `https://vault.bitwarden.eu` (or your self-hosted URL) | +| `bws timed out` | Network blocked or Bitwarden API slow | Check connectivity to `api.bitwarden.com` (or your `server_url`) | | `bws binary not available` | `auto_install: false` and `bws` not on PATH | Install manually from [github.com/bitwarden/sdk-sm/releases](https://github.com/bitwarden/sdk-sm/releases) or flip `auto_install` back on | | `Checksum mismatch` | Download corrupted or tampered | Re-run, will retry; if it persists, file an issue |