From bcfc928ff006108cbf70bab4433e3f3ad037de32 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Fri, 24 Jul 2026 08:24:05 -0700 Subject: [PATCH] feat(skills): add har-derived-api-client optional skill Record a site's XHR into a HAR with Playwright, derive its private JSON API, and call it directly over plain HTTP instead of browser-controlling the page every time. Credit: trick by Jared Longster, popularized by Dax (thdxr). - scripts/har_capture.py: Playwright HAR recorder with scripted --action steps and embedded response bodies - scripts/har_to_client.py: distills the HAR to endpoints (method/path template /params/body/response) plus User-Agent+cookie+auth replay hints - Validated live: derived + replayed the Algolia HN-search POST API and the Wikipedia rest.php search-title GET, both browserless - tests exercise the real derivation logic on a synthetic HAR fixture optional-skills placement: heavy Playwright dependency, niche use case. --- .../har-derived-api-client/SKILL.md | 122 ++++++++++++++ .../scripts/har_capture.py | 72 ++++++++ .../scripts/har_to_client.py | 147 ++++++++++++++++ .../test_har_derived_api_client_skill.py | 159 ++++++++++++++++++ 4 files changed, 500 insertions(+) create mode 100644 optional-skills/web-development/har-derived-api-client/SKILL.md create mode 100644 optional-skills/web-development/har-derived-api-client/scripts/har_capture.py create mode 100644 optional-skills/web-development/har-derived-api-client/scripts/har_to_client.py create mode 100644 tests/skills/test_har_derived_api_client_skill.py diff --git a/optional-skills/web-development/har-derived-api-client/SKILL.md b/optional-skills/web-development/har-derived-api-client/SKILL.md new file mode 100644 index 00000000000..1d70f630197 --- /dev/null +++ b/optional-skills/web-development/har-derived-api-client/SKILL.md @@ -0,0 +1,122 @@ +--- +name: har-derived-api-client +description: Record a site's XHR into a HAR, derive an HTTP client. +version: 0.1.0 +author: Hermes Agent +license: MIT +platforms: [linux, macos, windows] +metadata: + hermes: + tags: [Browser, HAR, API, Reverse-Engineering, Playwright] + category: web-development +--- + +# HAR-Derived API Client + +Drive a website once with a real browser while recording its network traffic +to a HAR file, then distill that HAR into the site's private JSON API so you +can call it directly with plain HTTP — far cheaper and faster than +browser-controlling the page on every request. Credit: trick by Jared Longster, +popularized by Dax (thdxr). This captures and replays; it does NOT bypass +auth, solve CAPTCHAs, or defeat bot-detection — if the site needs a logged-in +session, you carry its headers/cookies forward, you don't forge them. + +The two scripts are stdlib-plus-Playwright: capture needs Playwright, +derivation is pure stdlib, replay needs only `requests`/`httpx` (or `curl`). + +## When to Use + +- "Build a CLI/client for " — derive its API instead of scripting clicks. +- "This site has no public API but the page clearly fetches JSON." +- You're about to loop `browser_navigate` for the same query repeatedly — stop and derive the endpoint once. +- Reverse-engineering an autocomplete, search, feed, or checkout XHR. + +## Prerequisites + +- Playwright + a browser binary (capture step only): + - `pip install playwright` then `playwright install chromium` + - (If a system Playwright already has browsers under `~/.cache/ms-playwright`, reuse it.) +- `requests` or `httpx` for the replay step (stdlib `urllib` also works). +- No API keys. Any keys/tokens the client needs are the ones the HAR captured. + +## How to Run + +Two scripts under this skill's `scripts/`, both invoked through the `terminal` tool: + +1. `har_capture.py` — launches Chromium, runs your scripted interactions, writes a HAR with request+response bodies embedded. +2. `har_to_client.py` — filters the HAR to XHR/fetch/JSON, groups by endpoint, and prints params, headers, bodies, and replay hints (User-Agent / cookie / auth). + +Resolve paths against this skill's directory. Canonical loop: + +```bash +# 1. Capture — trigger the interaction whose network call you want +python3 scripts/har_capture.py "https://SITE/" out.har \ + --action "fill:input[name=search]:my query" --action "sleep:3" --wait 2 + +# 2. Derive — read the endpoints out of the HAR +python3 scripts/har_to_client.py out.har --host SITE --max-body 400 + +# 3. Replay — write a tiny client from the printed endpoint (see Procedure) +``` + +## Quick Reference + +``` +har_capture.py [--wait S] [--headed] [--action SPEC ...] + action SPEC: fill:SELECTOR:TEXT | press:SELECTOR:KEY | click:SELECTOR + goto:URL | sleep:SECONDS (run in order after page load) + +har_to_client.py [--host SUBSTR] [--include-static] [--max-body N] + default: keeps only XHR/fetch/JSON; --host narrows to one domain + prints per endpoint: query params, non-boring req headers, req body sample, + response status/content-type + body sample + prints "### Replay hints": the browser User-Agent, cookie/auth presence +``` + +## Procedure + +1. **Find the interaction.** Open the site with `browser_navigate` (or `--headed` capture) to see which selector to type into / click, and confirm a JSON XHR fires in devtools/network. +2. **Capture the HAR** via the `terminal` tool. Order `--action` to reach the request: `fill` the box, then `sleep` long enough for the debounced XHR, and always leave `--wait` at the end so late responses flush. `har_capture.py` records response bodies (`record_har_content="embed"`), so the derived client sees real payload shapes. +3. **Derive** with `har_to_client.py --host `. Read off: the method, the URL/path template (numeric/UUID segments collapse to `{id}`), query params, request-body JSON, and the `### Replay hints` block. +4. **Write the client.** Recreate the request exactly — same method, path, query params, body. Send the headers the site actually needs: at minimum copy the **User-Agent** from the replay hints. If hints report cookies or an auth/token header, resend those too. +5. **Test browserless.** Run the client with the `terminal` tool and confirm it returns the same data the browser saw. This is the payoff: no browser in the loop. +6. **(Optional) Wrap as a CLI** — a small `argparse` script over the derived call, e.g. `search.py "frank herbert"`. + +Worked example (Wikipedia search-title, derived + replayed live): + +```python +import requests +r = requests.get( + "https://en.wikipedia.org/w/rest.php/v1/search/title", + params={"q": "frank herbert", "limit": 5}, + headers={"accept": "application/json", + "User-Agent": "Mozilla/5.0 ... Chrome/131 Safari/537.36"}, # from HAR + timeout=15, +) +for p in r.json()["pages"]: + print(p["title"], "-", p.get("description")) +``` + +## Pitfalls + +- **Default library User-Agent gets 403.** Many sites (Wikipedia, Cloudflare-fronted APIs) reject `python-requests/x.y`. Always send the browser UA from the replay hints. This is the #1 reason a derived client fails when the browser succeeded. +- **A failed `--action` aborts before the HAR flushes** — you get no file. If capture errors on a selector, the run produced nothing; fix the selector (use `--headed` to watch) and rerun. Don't debug a missing HAR. +- **Server-rendered pages have no XHR** to derive — `har_to_client.py` prints "No API-looking entries". The data came in the HTML; scrape it or find the interaction that does fetch JSON. +- **Debounced/typeahead XHRs need a real pause.** Add `--action "sleep:3"` after `fill`; typing alone won't have fired the request when the HAR closes. +- **Auth/session endpoints** need the captured `Cookie`/`Authorization` header, and those expire. The derived client is only as durable as the credential; re-capture when it 401s. HARs contain live secrets — treat `out.har` as sensitive and delete it after deriving. +- **`record_har_content="embed"` makes big HARs.** Use `--max-body` to cap what's printed; the file itself can be large for media-heavy pages. +- **Endpoints shift.** Sites change private APIs without notice. Re-run the capture→derive loop when a client breaks rather than patching URLs by hand. + +## Verification + +End-to-end proof against a live site with no API key: + +```bash +python3 scripts/har_capture.py "https://en.wikipedia.org/wiki/Main_Page" /tmp/wiki.har \ + --action "fill:input[name=search]:dune messiah" --action "sleep:3" --wait 2 +python3 scripts/har_to_client.py /tmp/wiki.har --host wikipedia.org --max-body 200 +``` + +Expect the derivation to print `GET https://en.wikipedia.org/w/rest.php/v1/search/title` +with `q` and `limit` params and a JSON `pages` response — then replay it with the +Procedure snippet and confirm matching titles come back over plain HTTP. diff --git a/optional-skills/web-development/har-derived-api-client/scripts/har_capture.py b/optional-skills/web-development/har-derived-api-client/scripts/har_capture.py new file mode 100644 index 00000000000..3301bbc8c0e --- /dev/null +++ b/optional-skills/web-development/har-derived-api-client/scripts/har_capture.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Record a HAR file while driving a website with Playwright. + +Usage: + python3 har_capture.py [--wait SECONDS] \ + [--action "fill:SELECTOR:TEXT"] [--action "press:SELECTOR:KEY"] \ + [--action "click:SELECTOR"] [--action "goto:URL"] [--action "sleep:SECONDS"] + +Actions run in order after page load. The HAR embeds request/response bodies +(record_har_content='embed') so derived clients can see payload shapes. + +NOTE: a failing action raises before the HAR is flushed -- you get no file. +Fix the selector (try --headed to watch) and rerun. +""" +import argparse +import sys +import time + +from playwright.sync_api import sync_playwright + + +def run_action(page, spec: str) -> None: + parts = spec.split(":", 2) + kind = parts[0] + if kind == "fill": + page.fill(parts[1], parts[2]) + elif kind == "press": + page.press(parts[1], parts[2]) + elif kind == "click": + page.click(parts[1]) + elif kind == "goto": + page.goto(parts[1] + (":" + parts[2] if len(parts) > 2 else "")) + elif kind == "sleep": + time.sleep(float(parts[1])) + else: + raise ValueError(f"unknown action: {spec}") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("url") + ap.add_argument("har_path") + ap.add_argument("--wait", type=float, default=3.0, + help="seconds to idle at the end so late XHRs land in the HAR") + ap.add_argument("--action", action="append", default=[], + help="fill:SEL:TEXT | press:SEL:KEY | click:SEL | goto:URL | sleep:SECS") + ap.add_argument("--headed", action="store_true") + args = ap.parse_args() + + with sync_playwright() as p: + browser = p.chromium.launch(headless=not args.headed) + context = browser.new_context( + record_har_path=args.har_path, + record_har_content="embed", # keep response bodies in the HAR + ) + page = context.new_page() + page.goto(args.url, wait_until="domcontentloaded") + for spec in args.action: + run_action(page, spec) + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass # some pages never fully idle; the trailing --wait covers it + time.sleep(args.wait) + context.close() # flushes the HAR + browser.close() + print(f"HAR written: {args.har_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/optional-skills/web-development/har-derived-api-client/scripts/har_to_client.py b/optional-skills/web-development/har-derived-api-client/scripts/har_to_client.py new file mode 100644 index 00000000000..27eb2821a47 --- /dev/null +++ b/optional-skills/web-development/har-derived-api-client/scripts/har_to_client.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Distill a HAR file into an API summary an agent can turn into a client. + +Usage: + python3 har_to_client.py [--include-static] [--host SUBSTRING] [--max-body 600] + +Filters to XHR/fetch/JSON traffic by default, groups by (method, host, path +template), and prints per-endpoint: query params, interesting request headers, +request body sample, response content-type/status, and a response body sample. +Numeric/UUID-ish path segments are collapsed to {id} so repeated calls group. +Also prints "### Replay hints": the browser User-Agent plus whether cookies or +auth/token headers were present -- send those in the derived client or you may +get a 403/401. +""" +import argparse +import json +import re +import sys +from collections import OrderedDict +from urllib.parse import urlsplit + +BORING_HEADERS = { + "accept-encoding", "accept-language", "connection", "content-length", + "host", "origin", "referer", "sec-ch-ua", "sec-ch-ua-mobile", + "sec-ch-ua-platform", "sec-fetch-dest", "sec-fetch-mode", "sec-fetch-site", + "user-agent", "pragma", "cache-control", "priority", "te", + "upgrade-insecure-requests", "cookie", +} +ID_SEG = re.compile(r"^(\d+|[0-9a-f]{8}-[0-9a-f-]{27,}|[0-9a-f]{16,})$", re.I) +STATIC_EXT = re.compile(r"\.(js|css|png|jpe?g|gif|svg|webp|ico|woff2?|ttf|mp4|map)$", re.I) + + +def path_template(path: str) -> str: + segs = path.split("/") + return "/".join("{id}" if ID_SEG.match(s) else s for s in segs) + + +def is_api_entry(entry: dict) -> bool: + req = entry["request"] + resp = entry.get("response", {}) + rtype = (entry.get("_resourceType") or "").lower() + mime = (resp.get("content", {}).get("mimeType") or "").lower() + if rtype in ("xhr", "fetch"): + return True + if "json" in mime: + return True + if req["method"] not in ("GET", "HEAD") and not STATIC_EXT.search(urlsplit(req["url"]).path): + return True + return False + + +def trunc(text, n: int) -> str: + text = text if isinstance(text, str) else str(text) + return text if len(text) <= n else text[:n] + f"... [{len(text)} chars total]" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("har") + ap.add_argument("--include-static", action="store_true") + ap.add_argument("--host", default=None, help="only endpoints whose host contains this") + ap.add_argument("--max-body", type=int, default=600) + args = ap.parse_args() + + with open(args.har, encoding="utf-8") as f: + har = json.load(f) + + groups = OrderedDict() + for entry in har["log"]["entries"]: + req = entry["request"] + url = urlsplit(req["url"]) + if url.scheme not in ("http", "https"): + continue + if args.host and args.host not in url.netloc: + continue + if not args.include_static: + if STATIC_EXT.search(url.path) or not is_api_entry(entry): + continue + key = (req["method"], url.netloc, path_template(url.path)) + g = groups.setdefault(key, {"count": 0, "queries": set(), "headers": {}, + "req_body": None, "resp": None}) + g["count"] += 1 + for q in req.get("queryString", []): + g["queries"].add((q["name"], trunc(q["value"], 80))) + for h in req.get("headers", []): + name = h["name"].lower().lstrip(":") + if name in BORING_HEADERS or name in ("method", "path", "scheme", "authority"): + continue + g["headers"][name] = trunc(h["value"], 120) + post = req.get("postData", {}) + if post.get("text") and g["req_body"] is None: + g["req_body"] = (post.get("mimeType", ""), trunc(post["text"], args.max_body)) + resp = entry.get("response", {}) + if g["resp"] is None and resp: + content = resp.get("content", {}) + g["resp"] = (resp.get("status"), content.get("mimeType", ""), + trunc(content.get("text") or "", args.max_body)) + + if not groups: + print("No API-looking entries found. Re-run with --include-static to see everything.") + return 1 + + # Surface the browser identity so the replay client can match it (many + # sites 403 a default library User-Agent). + ua = None + saw_cookie = saw_auth = False + for entry in har["log"]["entries"]: + for h in entry["request"].get("headers", []): + n = h["name"].lower() + if n == "user-agent" and ua is None: + ua = h["value"] + if n == "cookie": + saw_cookie = True + if n in ("authorization", "x-api-key") or "token" in n: + saw_auth = True + print("### Replay hints") + if ua: + print(f" User-Agent (send this): {ua}") + if saw_cookie: + print(" Cookies present -> session may be auth-gated; capture & resend the Cookie header.") + if saw_auth: + print(" Authorization/token header present -> extract and resend it.") + + for (method, host, path), g in groups.items(): + print(f"\n=== {method} https://{host}{path} (x{g['count']})") + if g["queries"]: + print(" query params:") + for name, val in sorted(g["queries"]): + print(f" {name} = {val}") + if g["headers"]: + print(" request headers (non-boring):") + for name, val in sorted(g["headers"].items()): + print(f" {name}: {val}") + if g["req_body"]: + print(f" request body ({g['req_body'][0]}):") + print(f" {g['req_body'][1]}") + if g["resp"]: + status, mime, body = g["resp"] + print(f" response: {status} {mime}") + if body: + print(f" {body}") + print(f"\n{len(groups)} distinct endpoints.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/skills/test_har_derived_api_client_skill.py b/tests/skills/test_har_derived_api_client_skill.py new file mode 100644 index 00000000000..1e5278a94aa --- /dev/null +++ b/tests/skills/test_har_derived_api_client_skill.py @@ -0,0 +1,159 @@ +"""Tests for the har-derived-api-client optional skill. + +Two layers, both stdlib + pytest, no network: + 1. Structural / frontmatter contract on SKILL.md (matches the maintainer + review checklist for optional skills). + 2. Behavioral: run the real har_to_client.py logic against a synthetic HAR + fixture and assert it derives the endpoint, collapses id path segments, + filters static assets, and surfaces the User-Agent replay hint. +""" + +import importlib.util +import json +import re +from pathlib import Path + +import pytest + +SKILL_DIR = ( + Path(__file__).resolve().parents[2] + / "optional-skills" + / "web-development" + / "har-derived-api-client" +) +SKILL_MD = SKILL_DIR / "SKILL.md" +CAPTURE = SKILL_DIR / "scripts" / "har_capture.py" +DERIVE = SKILL_DIR / "scripts" / "har_to_client.py" + + +@pytest.fixture(scope="module") +def skill_text() -> str: + return SKILL_MD.read_text(encoding="utf-8") + + +def _load_module(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# --- structural contract --------------------------------------------------- + + +def test_skill_files_exist(): + assert SKILL_MD.is_file() + assert CAPTURE.is_file() + assert DERIVE.is_file() + + +def test_frontmatter_present(skill_text: str): + assert skill_text.startswith("---\n") + assert skill_text.count("---") >= 2 + + +def test_description_under_sixty_chars(skill_text: str): + m = re.search(r"^description: (.*)$", skill_text, re.MULTILINE) + assert m, "no description field" + desc = m.group(1).strip() + assert len(desc) <= 60, f"description is {len(desc)} chars (>60): {desc!r}" + assert desc.endswith("."), "description should end with a period" + + +def test_required_sections_present(skill_text: str): + for heading in ( + "## When to Use", + "## Prerequisites", + "## How to Run", + "## Quick Reference", + "## Procedure", + "## Pitfalls", + "## Verification", + ): + assert heading in skill_text, f"missing section: {heading}" + + +# --- behavioral: derivation logic ----------------------------------------- + + +def _make_har() -> dict: + return { + "log": { + "entries": [ + { # a JSON API call we want derived, with an id path segment + "_resourceType": "fetch", + "request": { + "method": "GET", + "url": "https://api.example.com/v1/items/12345/reviews?limit=5", + "queryString": [{"name": "limit", "value": "5"}], + "headers": [ + {"name": "User-Agent", "value": "Mozilla/5.0 TestBrowser/1.0"}, + {"name": "accept", "value": "application/json"}, + {"name": "referer", "value": "https://example.com/"}, + ], + }, + "response": { + "status": 200, + "content": { + "mimeType": "application/json", + "text": '{"reviews":[{"id":1}]}', + }, + }, + }, + { # a static asset we must filter out by default + "_resourceType": "script", + "request": { + "method": "GET", + "url": "https://cdn.example.com/app.js", + "queryString": [], + "headers": [{"name": "User-Agent", "value": "Mozilla/5.0 TestBrowser/1.0"}], + }, + "response": {"status": 200, "content": {"mimeType": "application/javascript"}}, + }, + ] + } + } + + +def test_derives_endpoint_and_filters_static(tmp_path, capsys): + mod = _load_module(DERIVE, "har_to_client_undertest") + har = tmp_path / "t.har" + har.write_text(json.dumps(_make_har()), encoding="utf-8") + + import sys + + argv = sys.argv + try: + sys.argv = ["har_to_client.py", str(har), "--host", "example.com"] + rc = mod.main() + finally: + sys.argv = argv + out = capsys.readouterr().out + + assert rc == 0 + # id path segment collapsed to {id} + assert "GET https://api.example.com/v1/items/{id}/reviews" in out + # query param surfaced + assert "limit = 5" in out + # static JS filtered out + assert "app.js" not in out + # boring header dropped, useful one absent from list but UA promoted to hints + assert "referer" not in out + # replay hint carries the browser UA + assert "User-Agent (send this): Mozilla/5.0 TestBrowser/1.0" in out + + +def test_path_template_collapses_ids(): + mod = _load_module(DERIVE, "har_to_client_undertest2") + assert mod.path_template("/v1/items/12345/x") == "/v1/items/{id}/x" + assert mod.path_template("/v1/items/abc/x") == "/v1/items/abc/x" + + +def test_capture_actions_parse_ok(): + # har_capture imports playwright at module top; only assert the file is + # syntactically valid and exposes run_action without importing playwright. + src = CAPTURE.read_text(encoding="utf-8") + compile(src, str(CAPTURE), "exec") + assert "def run_action(" in src + assert 'record_har_content="embed"' in src