mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-16 14:32:34 +00:00
c2828f2b9b
unbroker finds where a consenting person's info is exposed across data brokers and people-search sites and files the removals, running as far as each site allows and handing only genuinely human-only steps (hard CAPTCHA, gov-ID, phone, fax) back as an end-of-run digest. - Deterministic stdlib CLI (scripts/pdd.py) owns config, dossiers+consent, the broker DB, tier planning, the ledger, email, and the autonomous action queue; the agent scans/submits with native tools (web_extract, browser_*, delegate_task, cronjob, terminal). - Verify-before-disclose, least-disclosure (never volunteers SSN), consent gate, opaque ids, optional age-at-rest encryption, file-locked ledger. - Jurisdiction-aware (CCPA/CPRA, GDPR, generic); CA DROP one-shot covers the state registry (~545) in a single request; BADBOOL + curated people-search coverage; scheduled re-scan for re-listing. - No CAPTCHA-solving services or anti-bot bypass; browser email mode needs no stored password. - 85 hermetic tests (tests/skills/test_unbroker_skill.py; SMTP/IMAP via injected fakes, registry via CSV fixtures). Ships placeholder data only. Broker dataset adapted from BADBOOL (Yael Grauer, CC BY-NC-SA 4.0).
32 lines
1.3 KiB
Python
32 lines
1.3 KiB
Python
"""Stdlib fetch helper for simple url_pattern brokers (osint-style).
|
|
|
|
For JS-rendered or anti-bot pages the agent should use the `web_extract` or
|
|
`browser_navigate` tools (and the `scrapling` skill for stealth/Cloudflare).
|
|
This helper only covers plain static pages and is intentionally network-light so
|
|
it can be mocked in tests.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
USER_AGENT = "Mozilla/5.0 (compatible; unbroker/1.0; data opt-out)"
|
|
|
|
|
|
def fetch(url: str, timeout: int = 20) -> tuple[int, str]:
|
|
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 (https only by convention)
|
|
charset = resp.headers.get_content_charset() or "utf-8"
|
|
return getattr(resp, "status", 200), resp.read().decode(charset, errors="replace")
|
|
except urllib.error.HTTPError as exc:
|
|
return exc.code, ""
|
|
except (urllib.error.URLError, TimeoutError, ValueError):
|
|
return 0, ""
|
|
|
|
|
|
def looks_listed(html: str, match_signal: str | None) -> bool:
|
|
"""Naive confirmation heuristic for static pages: does the match signal appear?"""
|
|
if not html or not match_signal:
|
|
return False
|
|
return match_signal.lower() in html.lower()
|