From baf9ac281f76f813b0cd0b2e02174ed249283b9a Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:15:20 -0700 Subject: [PATCH] feat(skills): cover all Hermes browser pathways in har-derived-api-client Adds scripts/har_capture_cdp.py for browsers reached over CDP -- cloud backends (Browserbase, Browser-Use, Firecrawl), Camofox-with-CDP, and any /browser connect endpoint. record_har_path only works on a locally-owned Playwright context, so the CDP capturer attaches via connect_over_cdp() and assembles the HAR from page request/response events instead, leaving the attached browser open (it doesn't own it). - SKILL.md: pathway->capturer routing table, CDP prerequisites, pitfalls for wrong-capturer/empty-HAR, headless-UA weakness, and no-close-on-attach - Validated live: attached to an external CDP Chrome, drove DuckDuckGo autocomplete, derived the /ac/ endpoint, replayed it browserless - tests: assert CDP capturer attaches (not launches) and that the skill documents every browser backend --- .../har-derived-api-client/SKILL.md | 55 ++++++- .../scripts/har_capture_cdp.py | 135 ++++++++++++++++++ .../test_har_derived_api_client_skill.py | 21 +++ 3 files changed, 204 insertions(+), 7 deletions(-) create mode 100644 optional-skills/web-development/har-derived-api-client/scripts/har_capture_cdp.py diff --git a/optional-skills/web-development/har-derived-api-client/SKILL.md b/optional-skills/web-development/har-derived-api-client/SKILL.md index 1d70f630197..3bd98c74dab 100644 --- a/optional-skills/web-development/har-derived-api-client/SKILL.md +++ b/optional-skills/web-development/har-derived-api-client/SKILL.md @@ -21,8 +21,14 @@ popularized by Dax (thdxr). This captures and replays; it does NOT bypass auth, solve CAPTCHAs, or defeat bot-detection — if the site needs a logged-in session, you carry its headers/cookies forward, you don't forge them. -The two scripts are stdlib-plus-Playwright: capture needs Playwright, -derivation is pure stdlib, replay needs only `requests`/`httpx` (or `curl`). +The scripts are stdlib-plus-Playwright: capture needs Playwright, derivation +is pure stdlib, replay needs only `requests`/`httpx` (or `curl`). + +Covers **every Hermes browser pathway**: the default local `browser_navigate` +backend, plus the cloud/remote backends (Browserbase, Browser-Use, Firecrawl) +and any `/browser connect` CDP endpoint. There are two capture scripts — one +for a browser you launch, one for a browser you attach to over CDP — because +HAR recording works differently in each case (see How to Run). ## When to Use @@ -30,6 +36,7 @@ derivation is pure stdlib, replay needs only `requests`/`httpx` (or `curl`). - "This site has no public API but the page clearly fetches JSON." - You're about to loop `browser_navigate` for the same query repeatedly — stop and derive the endpoint once. - Reverse-engineering an autocomplete, search, feed, or checkout XHR. +- You captured a session on a cloud backend (Browserbase / Browser-Use / Firecrawl) or via `/browser connect` and want the API without re-renting the browser. ## Prerequisites @@ -38,21 +45,46 @@ derivation is pure stdlib, replay needs only `requests`/`httpx` (or `curl`). - (If a system Playwright already has browsers under `~/.cache/ms-playwright`, reuse it.) - `requests` or `httpx` for the replay step (stdlib `urllib` also works). - No API keys. Any keys/tokens the client needs are the ones the HAR captured. +- For the CDP path (`har_capture_cdp.py`): a reachable CDP endpoint. On Hermes, + run `/browser connect` to print the active endpoint, or read `BROWSER_CDP_URL` + / `browser.cdp_url` in config. Cloud backends expose it as `cdpUrl`/`connectUrl`. ## How to Run -Two scripts under this skill's `scripts/`, both invoked through the `terminal` tool: +Scripts under this skill's `scripts/`, invoked through the `terminal` tool. +**Pick the capturer by pathway** — this is the part that trips people up: -1. `har_capture.py` — launches Chromium, runs your scripted interactions, writes a HAR with request+response bodies embedded. -2. `har_to_client.py` — filters the HAR to XHR/fetch/JSON, groups by endpoint, and prints params, headers, bodies, and replay hints (User-Agent / cookie / auth). +| Browser pathway | How Hermes reaches it | Capturer | +|---|---|---| +| Local `browser_navigate` (default, agent-browser/Playwright) | launched locally | `har_capture.py` | +| Camofox (`CAMOFOX_URL` set) | local REST/CDP | `har_capture_cdp.py` if it exposes CDP, else drive it yourself | +| Browserbase / Browser-Use / Firecrawl (cloud) | **CDP** (`cdpUrl`) | `har_capture_cdp.py` | +| `/browser connect ` / `BROWSER_CDP_URL` | **CDP** | `har_capture_cdp.py` | + +Rule of thumb: **if Hermes *launched* the browser, use `har_capture.py`; if it +*connected to* one over CDP, use `har_capture_cdp.py`.** `har_capture.py` uses +Playwright's `record_har_path`, which only works on a locally-owned context. +`har_capture_cdp.py` attaches with `connect_over_cdp()` and assembles the HAR +from `page.on("request"/"response")` events, because `record_har_path` is +unavailable on a connected browser. + +Then, for either path: + +- `har_to_client.py` — filters the HAR to XHR/fetch/JSON, groups by endpoint, and prints params, headers, bodies, and replay hints (User-Agent / cookie / auth). Resolve paths against this skill's directory. Canonical loop: ```bash -# 1. Capture — trigger the interaction whose network call you want +# 1a. Capture, LOCAL browser (Hermes launched it) python3 scripts/har_capture.py "https://SITE/" out.har \ --action "fill:input[name=search]:my query" --action "sleep:3" --wait 2 +# 1b. Capture, CDP browser (cloud backend or /browser connect) +# get the endpoint from /browser connect or BROWSER_CDP_URL +python3 scripts/har_capture_cdp.py "ws://HOST/devtools/browser/..." out.har \ + --goto "https://SITE/" --action "fill:input[name=search]:my query" \ + --action "sleep:3" --wait 2 + # 2. Derive — read the endpoints out of the HAR python3 scripts/har_to_client.py out.har --host SITE --max-body 400 @@ -65,6 +97,11 @@ python3 scripts/har_to_client.py out.har --host SITE --max-body 400 har_capture.py [--wait S] [--headed] [--action SPEC ...] action SPEC: fill:SELECTOR:TEXT | press:SELECTOR:KEY | click:SELECTOR goto:URL | sleep:SECONDS (run in order after page load) + use when Hermes LAUNCHED the browser (local browser_navigate default) + +har_capture_cdp.py [--goto URL] [--wait S] [--action SPEC ...] + same action SPEC; attaches to an existing CDP browser and does NOT close it + use for cloud backends (Browserbase/Browser-Use/Firecrawl) & /browser connect har_to_client.py [--host SUBSTR] [--include-static] [--max-body N] default: keeps only XHR/fetch/JSON; --host narrows to one domain @@ -75,8 +112,9 @@ har_to_client.py [--host SUBSTR] [--include-static] [--max-body N] ## Procedure +0. **Pick the capturer by pathway** (see How to Run table). Launched-locally → `har_capture.py`; reached over CDP → `har_capture_cdp.py`. On Hermes, `/browser connect` tells you the CDP endpoint when a cloud/remote backend is active. 1. **Find the interaction.** Open the site with `browser_navigate` (or `--headed` capture) to see which selector to type into / click, and confirm a JSON XHR fires in devtools/network. -2. **Capture the HAR** via the `terminal` tool. Order `--action` to reach the request: `fill` the box, then `sleep` long enough for the debounced XHR, and always leave `--wait` at the end so late responses flush. `har_capture.py` records response bodies (`record_har_content="embed"`), so the derived client sees real payload shapes. +2. **Capture the HAR** via the `terminal` tool. Order `--action` to reach the request: `fill` the box, then `sleep` long enough for the debounced XHR, and always leave `--wait` at the end so late responses flush. Both capturers embed response bodies, so the derived client sees real payload shapes. 3. **Derive** with `har_to_client.py --host `. Read off: the method, the URL/path template (numeric/UUID segments collapse to `{id}`), query params, request-body JSON, and the `### Replay hints` block. 4. **Write the client.** Recreate the request exactly — same method, path, query params, body. Send the headers the site actually needs: at minimum copy the **User-Agent** from the replay hints. If hints report cookies or an auth/token header, resend those too. 5. **Test browserless.** Run the client with the `terminal` tool and confirm it returns the same data the browser saw. This is the payoff: no browser in the loop. @@ -106,6 +144,9 @@ for p in r.json()["pages"]: - **Auth/session endpoints** need the captured `Cookie`/`Authorization` header, and those expire. The derived client is only as durable as the credential; re-capture when it 401s. HARs contain live secrets — treat `out.har` as sensitive and delete it after deriving. - **`record_har_content="embed"` makes big HARs.** Use `--max-body` to cap what's printed; the file itself can be large for media-heavy pages. - **Endpoints shift.** Sites change private APIs without notice. Re-run the capture→derive loop when a client breaks rather than patching URLs by hand. +- **Wrong capturer = empty/no HAR.** `har_capture.py` on a cloud/CDP backend records nothing (it launches its own local browser instead of the one you meant). `har_capture_cdp.py` needs the endpoint; on Hermes get it from `/browser connect` or `BROWSER_CDP_URL`. Match the capturer to the pathway (How to Run table). +- **Headless-Chrome UA is a weak tell.** Local/agent-browser capture yields a `HeadlessChrome/...` User-Agent; some sites sniff the "Headless" token. Cloud backends (Browserbase/Browser-Use) send a real desktop-Chrome UA, so a client derived from a cloud capture replays more reliably. If a headless-derived client 403s where the browser didn't, swap the "Headless" UA for a normal Chrome UA string before assuming the endpoint changed. +- **CDP capture doesn't close the browser.** `har_capture_cdp.py` attaches to a browser it doesn't own and leaves it running — correct for cloud/remote sessions Hermes manages. Don't add a close; let the owning backend tear it down. ## Verification diff --git a/optional-skills/web-development/har-derived-api-client/scripts/har_capture_cdp.py b/optional-skills/web-development/har-derived-api-client/scripts/har_capture_cdp.py new file mode 100644 index 00000000000..cc6facc1ce7 --- /dev/null +++ b/optional-skills/web-development/har-derived-api-client/scripts/har_capture_cdp.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Capture a HAR from a browser you connect to over CDP (not one you launch). + +Use this when the browser is owned by someone else and only reachable over the +Chrome DevTools Protocol: Hermes cloud backends (Browserbase, Browser-Use, +Firecrawl), a Camofox session exposing CDP, or anything wired via +`/browser connect ` / BROWSER_CDP_URL / browser.cdp_url in config. + +Why this exists: Playwright's record_har_path only works on a context you +launched locally. connect_over_cdp() attaches to an existing browser, so +record_har is unavailable — we assemble the HAR from CDP Network.* events +ourselves via page.on("request"/"response"). + +Usage: + python3 har_capture_cdp.py [--wait S] \ + [--goto URL] [--action "fill:SEL:TEXT"] [--action "click:SEL"] ... + + is the ws:// or http:// CDP endpoint. For Hermes: run +`/browser connect` to see the active endpoint, or read BROWSER_CDP_URL. +""" +import argparse +import base64 +import json +import sys +import time + +from playwright.sync_api import sync_playwright + + +def run_action(page, spec: str) -> None: + parts = spec.split(":", 2) + kind = parts[0] + if kind == "fill": + page.fill(parts[1], parts[2]) + elif kind == "press": + page.press(parts[1], parts[2]) + elif kind == "click": + page.click(parts[1]) + elif kind == "goto": + page.goto(parts[1] + (":" + parts[2] if len(parts) > 2 else "")) + elif kind == "sleep": + time.sleep(float(parts[1])) + else: + raise ValueError(f"unknown action: {spec}") + + +def _har_entry(req, resp): + """Build a minimal HAR entry from a Playwright request/response pair.""" + body_text, encoding = "", "" + if resp is not None: + try: + raw = resp.body() + try: + body_text = raw.decode("utf-8") + except UnicodeDecodeError: + body_text = base64.b64encode(raw).decode("ascii") + encoding = "base64" + except Exception: + pass + post = req.post_data + return { + "_resourceType": req.resource_type, + "request": { + "method": req.method, + "url": req.url, + "headers": [{"name": k, "value": v} for k, v in req.headers.items()], + "queryString": [], # har_to_client.py re-parses the URL, so leave empty + "postData": {"mimeType": req.headers.get("content-type", ""), + "text": post} if post else {}, + }, + "response": { + "status": resp.status if resp else 0, + "headers": [{"name": k, "value": v} for k, v in (resp.headers.items() if resp else [])], + "content": { + "mimeType": (resp.headers.get("content-type", "") if resp else ""), + "text": body_text, + **({"encoding": encoding} if encoding else {}), + }, + }, + } + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("cdp_url") + ap.add_argument("har_path") + ap.add_argument("--goto", default=None, help="URL to navigate to after attaching") + ap.add_argument("--wait", type=float, default=3.0) + ap.add_argument("--action", action="append", default=[]) + args = ap.parse_args() + + entries = [] + pending = {} # id(request) -> request + + with sync_playwright() as p: + browser = p.chromium.connect_over_cdp(args.cdp_url) + context = browser.contexts[0] if browser.contexts else browser.new_context() + page = context.pages[0] if context.pages else context.new_page() + + def on_request(req): + pending[id(req)] = req + + def on_response(resp): + req = resp.request + pending.pop(id(req), None) + entries.append(_har_entry(req, resp)) + + page.on("request", on_request) + page.on("response", on_response) + + if args.goto: + page.goto(args.goto, wait_until="domcontentloaded") + for spec in args.action: + run_action(page, spec) + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass + time.sleep(args.wait) + + page.remove_listener("request", on_request) + page.remove_listener("response", on_response) + # Do NOT close: we connected to someone else's browser. + + har = {"log": {"version": "1.2", + "creator": {"name": "har_capture_cdp", "version": "0.1"}, + "entries": entries}} + with open(args.har_path, "w", encoding="utf-8") as f: + json.dump(har, f) + print(f"HAR written: {args.har_path} ({len(entries)} entries)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/skills/test_har_derived_api_client_skill.py b/tests/skills/test_har_derived_api_client_skill.py index 1e5278a94aa..f2f1fb18172 100644 --- a/tests/skills/test_har_derived_api_client_skill.py +++ b/tests/skills/test_har_derived_api_client_skill.py @@ -23,6 +23,7 @@ SKILL_DIR = ( ) SKILL_MD = SKILL_DIR / "SKILL.md" CAPTURE = SKILL_DIR / "scripts" / "har_capture.py" +CAPTURE_CDP = SKILL_DIR / "scripts" / "har_capture_cdp.py" DERIVE = SKILL_DIR / "scripts" / "har_to_client.py" @@ -45,6 +46,7 @@ def _load_module(path: Path, name: str): def test_skill_files_exist(): assert SKILL_MD.is_file() assert CAPTURE.is_file() + assert CAPTURE_CDP.is_file() assert DERIVE.is_file() @@ -157,3 +159,22 @@ def test_capture_actions_parse_ok(): compile(src, str(CAPTURE), "exec") assert "def run_action(" in src assert 'record_har_content="embed"' in src + + +def test_cdp_capture_is_valid_and_attaches_not_launches(): + # Covers the CDP pathway (cloud backends / /browser connect). Syntax-check + # without importing playwright, and assert it attaches (connect_over_cdp) + # and does NOT close a browser it doesn't own. + src = CAPTURE_CDP.read_text(encoding="utf-8") + compile(src, str(CAPTURE_CDP), "exec") + assert "connect_over_cdp(" in src + assert 'page.on("request"' in src and 'page.on("response"' in src + # must not tear down a browser it merely attached to + assert "browser.close()" not in src + + +def test_skill_documents_all_browser_pathways(skill_text: str): + # The skill must route every Hermes browser backend to the right capturer. + for token in ("Browserbase", "Browser-Use", "Firecrawl", "browser connect", + "har_capture_cdp.py", "connect_over_cdp"): + assert token in skill_text, f"pathway coverage missing: {token}"